Add a framework for detecting databases that need updating.

Put transaction from account_tx command in transaction application sequence.
CAUTION: This modifies your databases and will cause a delay on your next startup.

newcoin.vcxproj

@@ -186,6 +186,7 @@
     <ClCompile Include="src\cpp\ripple\Transactor.cpp" />
     <ClCompile Include="src\cpp\ripple\TrustSetTransactor.cpp" />
     <ClCompile Include="src\cpp\ripple\UniqueNodeList.cpp" />
+    <ClCompile Include="src\cpp\ripple\UpdateTables.cpp" />
     <ClCompile Include="src\cpp\ripple\utils.cpp" />
     <ClCompile Include="src\cpp\ripple\ValidationCollection.cpp" />
     <ClCompile Include="src\cpp\ripple\Wallet.cpp" />

newcoin.vcxproj.filters

@@ -270,6 +270,9 @@
     <ClCompile Include="src\cpp\ripple\UniqueNodeList.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="src\cpp\ripple\UpdateTables.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="src\cpp\ripple\utils.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>

ripple2010.vcxproj

@@ -185,6 +185,7 @@
     <ClCompile Include="src\cpp\ripple\Transactor.cpp" />
     <ClCompile Include="src\cpp\ripple\TrustSetTransactor.cpp" />
     <ClCompile Include="src\cpp\ripple\UniqueNodeList.cpp" />
+    <ClCompile Include="src\cpp\ripple\UpdateTables.cpp" />
     <ClCompile Include="src\cpp\ripple\utils.cpp" />
     <ClCompile Include="src\cpp\ripple\ValidationCollection.cpp" />
     <ClCompile Include="src\cpp\ripple\Wallet.cpp" />

ripple2010.vcxproj.filters

@@ -267,6 +267,9 @@
     <ClCompile Include="src\cpp\ripple\UniqueNodeList.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="src\cpp\ripple\UpdateTables.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="src\cpp\ripple\utils.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>

src/cpp/ripple/AcceptedLedger.h

@@ -31,6 +31,7 @@ public:
 	uint256 getTransactionID() const						{ return mTxn->getTransactionID(); }
 	TransactionType getTxnType() const						{ return mTxn->getTxnType(); }
 	TER getResult() const									{ return mResult; }
+	uint32 getTxnSeq() const								{ return mMeta->getIndex(); }
 
 	bool isApplied() const									{ return !!mMeta; }
 	int getIndex() const									{ return mMeta ? mMeta->getIndex() : 0; }

src/cpp/ripple/Application.cpp

@@ -143,6 +143,9 @@ void Application::setup()
 	mLedgerDB->getDB()->setupCheckpointing(&mJobQueue);
 	mHashNodeDB->getDB()->setupCheckpointing(&mJobQueue);
 
+	if (!theConfig.RUN_STANDALONE)
+		updateTables();
+
 	if (theConfig.START_UP == Config::FRESH)
 	{
 		cLog(lsINFO) << "Starting new Ledger";

src/cpp/ripple/Application.h

@@ -90,6 +90,7 @@ class Application
 
 	volatile bool			mShutdown;
 
+	void updateTables();
 	void startNewLedger();
 	void loadOldLedger(const std::string&);
 

src/cpp/ripple/DBInit.cpp

@@ -26,11 +26,12 @@ const char *TxnDBInit[] = {
 		TransID		CHARACTER(64),				\
 		Account		CHARACTER(64),				\
 		LedgerSeq	BIGINT UNSIGNED				\
+		TxnSeq		INTEGER						\
 	);",
 	"CREATE INDEX AcctTxIDIndex ON				\
 		AccountTransactions(TransID);",
 	"CREATE INDEX AcctTxIndex ON				\
-		AccountTransactions(Account, LedgerSeq, TransID);",
+		AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);",
 	"CREATE INDEX AcctLgrIndex ON				\
 		AccountTransactions(LedgerSeq, Account, TransID);",
 

src/cpp/ripple/Ledger.cpp

@@ -479,7 +479,7 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus)
 			const std::vector<RippleAddress>& accts = vt.second.getAffected();
 			if (!accts.empty())
 			{
-				std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq) VALUES ";
+				std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq, TxnSeq) VALUES ";
 				bool first = true;
 				for (std::vector<RippleAddress>::const_iterator it = accts.begin(), end = accts.end(); it != end; ++it)
 				{
@@ -495,6 +495,8 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus)
 					sql += it->humanAccountID();
 					sql += "',";
 					sql += boost::lexical_cast<std::string>(getLedgerSeq());
+					sql += ",";
+					sql += boost::lexical_cast<std::string>(vt.second.getTxnSeq());
 					sql += ")";
 				}
 				sql += ";";

src/cpp/ripple/LedgerMaster.cpp

@@ -106,7 +106,7 @@ Ledger::pointer LedgerMaster::closeLedger(bool recover)
 					++recovers;
 			}
 			catch (...)
-			{
+			{ // CHECKME: We got a few of these
 				cLog(lsWARNING) << "Held transaction throws";
 			}
 		}

src/cpp/ripple/NetworkOPs.cpp

@@ -1093,13 +1093,15 @@ std::string
 		boost::str(boost::format("SELECT %s FROM "
 		"AccountTransactions INNER JOIN Transactions ON Transactions.TransID = AccountTransactions.TransID "
 		"WHERE Account = '%s' %s %s "
-		"ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TransID %s LIMIT %u, %u;")
+		"ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s "
+		"LIMIT %u, %u;")
 		% selection
 		% account.humanAccountID()
 		% maxClause
 		% minClause
 		% (descending ? "DESC" : "ASC")
 		% (descending ? "DESC" : "ASC")
+		% (descending ? "DESC" : "ASC")
 		% boost::lexical_cast<std::string>(offset)
 		% boost::lexical_cast<std::string>(numberOfResults)
 		);

src/cpp/ripple/UpdateTables.cpp

@@ -0,0 +1,108 @@
+
+#include "Application.h"
+#include "Log.h"
+
+static std::vector<std::string> getSchema(DatabaseCon* dbc, const std::string& dbName)
+{
+	std::vector<std::string> schema;
+
+	std::string sql = "SELECT sql FROM sqlite_master WHERE tbl_name='";
+	sql += dbName;
+	sql += "';";
+
+	SQL_FOREACH(dbc->getDB(), sql)
+	{
+			dbc->getDB()->getStr("sql", sql);
+			schema.push_back(sql);
+	}
+
+	return schema;
+}
+
+static bool schemaHas(DatabaseCon* dbc, const std::string& dbName, int line, const std::string& content)
+{
+	std::vector<std::string> schema = getSchema(dbc, dbName);
+	if (schema.size() <= line)
+	{
+		Log(lsFATAL) << "Schema for " << dbName << " has too few lines";
+		throw std::runtime_error("bad schema");
+	}
+	return schema[line].find(content) != std::string::npos;
+}
+
+static void addTxnSeqField()
+{
+	if (schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TxnSeq"))
+		return;
+	Log(lsWARNING) << "Transaction sequence field is missing";
+
+	Database* db = theApp->getTxnDB()->getDB();
+
+	std::vector< std::pair<uint256, int> > txIDs;
+	txIDs.reserve(300000);
+
+	Log(lsINFO) << "Parsing transactions";
+	int i = 0;
+	uint256 transID;
+	SQL_FOREACH(db, "SELECT TransID,TxnMeta FROM Transactions;")
+	{
+		std::vector<unsigned char> rawMeta;
+		int metaSize = 2048;
+		rawMeta.resize(metaSize);
+		metaSize = db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size());
+		if (metaSize > rawMeta.size())
+		{
+			rawMeta.resize(metaSize);
+			db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size());
+		}
+		else rawMeta.resize(metaSize);
+
+		std::string tid;
+		db->getStr("TransID", tid);
+		transID.SetHex(tid, true);
+
+		if (rawMeta.size() == 0)
+		{
+			txIDs.push_back(std::make_pair(transID, -1));
+			Log(lsINFO) << "No metadata for " << transID;
+		}
+		else
+		{
+			TransactionMetaSet m(transID, 0, rawMeta);
+			txIDs.push_back(std::make_pair(transID, m.getIndex()));
+		}
+
+		if ((++i % 1000) == 0)
+			Log(lsINFO) << i << " transactions read";
+	}
+
+	Log(lsINFO) << "All " << i << " transactions read";
+
+	db->executeSQL("BEGIN TRANSACTION;");
+
+	Log(lsINFO) << "Dropping old index";
+	db->executeSQL("DROP INDEX AcctTxIndex;");
+
+	Log(lsINFO) << "Altering table";
+	db->executeSQL("ALTER TABLE AccountTransactions ADD COLUMN TxnSeq INTEGER;");
+
+	typedef std::pair<uint256, int> u256_int_pair_t;
+	boost::format fmt("UPDATE AccountTransactions SET TxnSeq = %d WHERE TransID = '%s';");
+	i = 0;
+	BOOST_FOREACH(u256_int_pair_t& t, txIDs)
+	{
+		db->executeSQL(boost::str(fmt % t.second % t.first.GetHex()));
+		if ((++i % 1000) == 0)
+			Log(lsINFO) << i << " transactions updated";
+	}
+
+	db->executeSQL("CREATE INDEX AcctTxIndex ON AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);");
+	db->executeSQL("END TRANSACTION;");
+}
+
+void Application::updateTables()
+{ // perform any needed table updates
+	assert(schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TransID"));
+	assert(!schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "foobar"));
+	addTxnSeqField();
+}