Merge branch 'master' of github.com:jedmccaleb/NewCoin

src/cpp/ripple/AcceptedLedger.h

@@ -31,6 +31,7 @@ public:
 	uint256 getTransactionID() const						{ return mTxn->getTransactionID(); }
 	TransactionType getTxnType() const						{ return mTxn->getTxnType(); }
 	TER getResult() const									{ return mResult; }
+	uint32 getTxnSeq() const								{ return mMeta->getIndex(); }
 
 	bool isApplied() const									{ return !!mMeta; }
 	int getIndex() const									{ return mMeta ? mMeta->getIndex() : 0; }

src/cpp/ripple/Application.cpp

@@ -143,6 +143,9 @@ void Application::setup()
 	mLedgerDB->getDB()->setupCheckpointing(&mJobQueue);
 	mHashNodeDB->getDB()->setupCheckpointing(&mJobQueue);
 
+	if (!theConfig.RUN_STANDALONE)
+		updateTables();
+
 	if (theConfig.START_UP == Config::FRESH)
 	{
 		cLog(lsINFO) << "Starting new Ledger";
@@ -416,12 +419,12 @@ void Application::loadOldLedger(const std::string& l)
 		mLedgerMaster.switchLedgers(loadLedger, openLedger);
 		mNetOps.setLastCloseTime(loadLedger->getCloseTimeNC());
 	}
-	catch (SHAMapMissingNode& mn)
+	catch (SHAMapMissingNode&)
 	{
 		cLog(lsFATAL) << "Data is missing for selected ledger";
 		exit(-1);
 	}
-	catch (boost::bad_lexical_cast& blc)
+	catch (boost::bad_lexical_cast&)
 	{
 		cLog(lsFATAL) << "Ledger specified '" << l << "' is not valid";
 		exit(-1);

src/cpp/ripple/Application.h

@@ -90,6 +90,7 @@ class Application
 
 	volatile bool			mShutdown;
 
+	void updateTables();
 	void startNewLedger();
 	void loadOldLedger(const std::string&);
 

src/cpp/ripple/DBInit.cpp

@@ -26,11 +26,12 @@ const char *TxnDBInit[] = {
 		TransID		CHARACTER(64),				\
 		Account		CHARACTER(64),				\
 		LedgerSeq	BIGINT UNSIGNED				\
+		TxnSeq		INTEGER						\
 	);",
 	"CREATE INDEX AcctTxIDIndex ON				\
 		AccountTransactions(TransID);",
 	"CREATE INDEX AcctTxIndex ON				\
-		AccountTransactions(Account, LedgerSeq, TransID);",
+		AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);",
 	"CREATE INDEX AcctLgrIndex ON				\
 		AccountTransactions(LedgerSeq, Account, TransID);",
 

src/cpp/ripple/HashedObject.cpp

@@ -95,7 +95,7 @@ void HashedObjectStore::bulkWrite()
 	{
 		Database* db = theApp->getHashNodeDB()->getDB();
 		ScopedLock sl(theApp->getHashNodeDB()->getDBLock());
-		static SqliteStatement pSt(db->getSqliteDB(),
+		SqliteStatement pSt(db->getSqliteDB(),
 			"INSERT OR IGNORE INTO CommittedObjects "
 				"(Hash,ObjType,LedgerIndex,Object) VALUES (?, ?, ?, ?);");
 
@@ -185,7 +185,7 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 	{
 		ScopedLock sl(theApp->getHashNodeDB()->getDBLock());
 		LoadEvent::autoptr event(theApp->getJobQueue().getLoadEventAP(jtDISK, "HOS::retrieve"));
-		static SqliteStatement pSt(theApp->getHashNodeDB()->getDB()->getSqliteDB(),
+		SqliteStatement pSt(theApp->getHashNodeDB()->getDB()->getSqliteDB(),
 			"SELECT ObjType,LedgerIndex,Object FROM CommittedObjects WHERE Hash = ?;");
 
 		pSt.bind(1, hash.GetHex());
@@ -193,7 +193,6 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 		int ret = pSt.step();
 		if (pSt.isDone(ret))
 		{
-			pSt.reset();
 			mNegativeCache.add(hash);
 			cLog(lsTRACE) << "HOS: " << hash <<" fetch: not in db";
 			return obj;
@@ -202,7 +201,6 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
 		type = pSt.peekString(0);
 		index = pSt.getUInt32(1);
 		pSt.getBlob(2).swap(data);
-		pSt.reset();
 	}
 
 #else
@@ -271,7 +269,7 @@ int HashedObjectStore::import(const std::string& file)
 		uint256 hash;
 		std::string hashStr;
 		importDB->getStr("Hash", hashStr);
-		hash.SetHex(hashStr, true);
+		hash.SetHexExact(hashStr);
 		if (hash.isZero())
 		{
 			cLog(lsWARNING) << "zero hash found in import table";

src/cpp/ripple/Ledger.cpp

@@ -479,7 +479,7 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus)
 			const std::vector<RippleAddress>& accts = vt.second.getAffected();
 			if (!accts.empty())
 			{
-				std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq) VALUES ";
+				std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq, TxnSeq) VALUES ";
 				bool first = true;
 				for (std::vector<RippleAddress>::const_iterator it = accts.begin(), end = accts.end(); it != end; ++it)
 				{
@@ -495,6 +495,8 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus)
 					sql += it->humanAccountID();
 					sql += "',";
 					sql += boost::lexical_cast<std::string>(getLedgerSeq());
+					sql += ",";
+					sql += boost::lexical_cast<std::string>(vt.second.getTxnSeq());
 					sql += ")";
 				}
 				sql += ";";
@@ -536,14 +538,13 @@ Ledger::pointer Ledger::loadByIndex(uint32 ledgerIndex)
 		Database* db = theApp->getLedgerDB()->getDB();
 		ScopedLock sl(theApp->getLedgerDB()->getDBLock());
 
-		static SqliteStatement pSt(db->getSqliteDB(), "SELECT "
+		SqliteStatement pSt(db->getSqliteDB(), "SELECT "
 			"LedgerHash,PrevHash,AccountSetHash,TransSetHash,TotalCoins,"
 			"ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags,LedgerSeq"
 			" from Ledgers WHERE LedgerSeq = ?;");
 
 		pSt.bind(1, ledgerIndex);
 		ledger = getSQL1(&pSt);
-		pSt.reset();
 	}
 	if (ledger)
 		Ledger::getSQL2(ledger);
@@ -557,14 +558,13 @@ Ledger::pointer Ledger::loadByHash(const uint256& ledgerHash)
 		Database* db = theApp->getLedgerDB()->getDB();
 		ScopedLock sl(theApp->getLedgerDB()->getDBLock());
 
-		static SqliteStatement pSt(db->getSqliteDB(), "SELECT "
+		SqliteStatement pSt(db->getSqliteDB(), "SELECT "
 			"LedgerHash,PrevHash,AccountSetHash,TransSetHash,TotalCoins,"
 			"ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags,LedgerSeq"
 			" from Ledgers WHERE LedgerHash = ?;");
 
 		pSt.bind(1, ledgerHash.GetHex());
 		ledger = getSQL1(&pSt);
-		pSt.reset();
 	}
 	if (ledger)
 	{
@@ -611,13 +611,13 @@ Ledger::pointer Ledger::getSQL(const std::string& sql)
 			return Ledger::pointer();
 
 		db->getStr("LedgerHash", hash);
-		ledgerHash.SetHex(hash, true);
+		ledgerHash.SetHexExact(hash);
 		db->getStr("PrevHash", hash);
-		prevHash.SetHex(hash, true);
+		prevHash.SetHexExact(hash);
 		db->getStr("AccountSetHash", hash);
-		accountHash.SetHex(hash, true);
+		accountHash.SetHexExact(hash);
 		db->getStr("TransSetHash", hash);
-		transHash.SetHex(hash, true);
+		transHash.SetHexExact(hash);
 		totCoins = db->getBigInt("TotalCoins");
 		closingTime = db->getBigInt("ClosingTime");
 		prevClosingTime = db->getBigInt("PrevClosingTime");
@@ -671,10 +671,10 @@ Ledger::pointer Ledger::getSQL1(SqliteStatement *stmt)
 	unsigned closeFlags;
 	std::string hash;
 
-	ledgerHash.SetHex(stmt->peekString(0), true);
-	prevHash.SetHex(stmt->peekString(1), true);
-	accountHash.SetHex(stmt->peekString(2), true);
-	transHash.SetHex(stmt->peekString(3), true);
+	ledgerHash.SetHexExact(stmt->peekString(0));
+	prevHash.SetHexExact(stmt->peekString(1));
+	accountHash.SetHexExact(stmt->peekString(2));
+	transHash.SetHexExact(stmt->peekString(3));
 	totCoins = stmt->getInt64(4);
 	closingTime = stmt->getUInt32(5);
 	prevClosingTime = stmt->getUInt32(6);
@@ -717,7 +717,7 @@ uint256 Ledger::getHashByIndex(uint32 ledgerIndex)
 		db->endIterRows();
 	}
 
-	ret.SetHex(hash, true);
+	ret.SetHexExact(hash);
 	return ret;
 }
 
@@ -728,7 +728,7 @@ bool Ledger::getHashesByIndex(uint32 ledgerIndex, uint256& ledgerHash, uint256&
 	DatabaseCon *con = theApp->getLedgerDB();
 	ScopedLock sl(con->getDBLock());
 
-	static SqliteStatement pSt(con->getDB()->getSqliteDB(),
+	SqliteStatement pSt(con->getDB()->getSqliteDB(),
 		"SELECT LedgerHash,PrevHash FROM Ledgers INDEXED BY SeqLedger Where LedgerSeq = ?;");
 
 	pSt.bind(1, ledgerIndex);
@@ -736,21 +736,18 @@ bool Ledger::getHashesByIndex(uint32 ledgerIndex, uint256& ledgerHash, uint256&
 	int ret = pSt.step();
 	if (pSt.isDone(ret))
 	{
-		pSt.reset();
 		cLog(lsTRACE) << "Don't have ledger " << ledgerIndex;
 		return false;
 	}
 	if (!pSt.isRow(ret))
 	{
-		pSt.reset();
 		assert(false);
 		cLog(lsFATAL) << "Unexpected statement result " << ret;
 		return false;
 	}
 
-	ledgerHash.SetHex(pSt.peekString(0), true);
-	parentHash.SetHex(pSt.peekString(1), true);
-	pSt.reset();
+	ledgerHash.SetHexExact(pSt.peekString(0));
+	parentHash.SetHexExact(pSt.peekString(1));
 
 	return true;
 
@@ -771,8 +768,8 @@ bool Ledger::getHashesByIndex(uint32 ledgerIndex, uint256& ledgerHash, uint256&
 		db->endIterRows();
 	}
 
-	ledgerHash.SetHex(hash, true);
-	parentHash.SetHex(prevHash, true);
+	ledgerHash.SetHexExact(hash);
+	parentHash.SetHexExact(prevHash);
 
 	assert(ledgerHash.isNonZero() && ((ledgerIndex == 0) || parentHash.isNonZero()));
 
@@ -788,7 +785,7 @@ std::map< uint32, std::pair<uint256, uint256> > Ledger::getHashesByIndex(uint32
 	DatabaseCon *con = theApp->getLedgerDB();
 	ScopedLock sl(con->getDBLock());
 
-	static SqliteStatement pSt(con->getDB()->getSqliteDB(),
+	SqliteStatement pSt(con->getDB()->getSqliteDB(),
 		"SELECT LedgerSeq,LedgerHash,PrevHash FROM Ledgers INDEXED BY SeqLedger "
 		"WHERE LedgerSeq >= ? AND LedgerSeq <= ?;");
 
@@ -801,17 +798,11 @@ std::map< uint32, std::pair<uint256, uint256> > Ledger::getHashesByIndex(uint32
 	{
 		int r = pSt.step();
 		if (pSt.isDone(r))
-		{
-			pSt.reset();
 			return ret;
-		}
 		if (!pSt.isRow(r))
-		{
-			pSt.reset();
 			return ret;
-		}
-		hashes.first.SetHex(pSt.peekString(1), true);
-		hashes.second.SetHex(pSt.peekString(2), true);
+		hashes.first.SetHexExact(pSt.peekString(1));
+		hashes.second.SetHexExact(pSt.peekString(2));
 		ret[pSt.getUInt32(0)] = hashes;
 	} while(1);
 

src/cpp/ripple/LedgerMaster.cpp

@@ -106,7 +106,7 @@ Ledger::pointer LedgerMaster::closeLedger(bool recover)
 					++recovers;
 			}
 			catch (...)
-			{
+			{ // CHECKME: We got a few of these
 				cLog(lsWARNING) << "Held transaction throws";
 			}
 		}

src/cpp/ripple/NetworkOPs.cpp

@@ -1093,13 +1093,15 @@ std::string
 		boost::str(boost::format("SELECT %s FROM "
 		"AccountTransactions INNER JOIN Transactions ON Transactions.TransID = AccountTransactions.TransID "
 		"WHERE Account = '%s' %s %s "
-		"ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TransID %s LIMIT %u, %u;")
+		"ORDER BY AccountTransactions.LedgerSeq %s, AccountTransactions.TxnSeq %s, AccountTransactions.TransID %s "
+		"LIMIT %u, %u;")
 		% selection
 		% account.humanAccountID()
 		% maxClause
 		% minClause
 		% (descending ? "DESC" : "ASC")
 		% (descending ? "DESC" : "ASC")
+		% (descending ? "DESC" : "ASC")
 		% boost::lexical_cast<std::string>(offset)
 		% boost::lexical_cast<std::string>(numberOfResults)
 		);

src/cpp/ripple/SNTPClient.cpp

@@ -81,7 +81,7 @@ void SNTPClient::resolveComplete(const boost::system::error_code& error, boost::
 			query.mReceivedReply = false;
 			query.mLocalTimeSent = now;
 			getRand(reinterpret_cast<unsigned char *>(&query.mQueryNonce), sizeof(query.mQueryNonce));
-			reinterpret_cast<uint32*>(SNTPQueryData)[NTP_OFF_XMITTS_INT] = time(NULL) + NTP_UNIX_OFFSET;
+			reinterpret_cast<uint32*>(SNTPQueryData)[NTP_OFF_XMITTS_INT] = static_cast<uint32>(time(NULL)) + NTP_UNIX_OFFSET;
 			reinterpret_cast<uint32*>(SNTPQueryData)[NTP_OFF_XMITTS_FRAC] = query.mQueryNonce;
 			mSocket.async_send_to(boost::asio::buffer(SNTPQueryData, 48), *sel,
 				boost::bind(&SNTPClient::sendComplete, this,
@@ -148,7 +148,7 @@ void SNTPClient::processReply()
 		return;
 	}
 
-	time_t now = time(NULL);
+	int64 now = static_cast<int>(time(NULL));
 	timev -= now;
 	timev -= NTP_UNIX_OFFSET;
 

src/cpp/ripple/TaggedCache.h

@@ -99,7 +99,7 @@ template<typename c_Key, typename c_Data> void TaggedCache<c_Key, c_Data>::setTa
 	boost::recursive_mutex::scoped_lock sl(mLock);
 	mTargetSize = s;
 	if (s > 0)
-		mCache.rehash((s + (s >> 2)) / mCache.max_load_factor() + 1);
+		mCache.rehash(static_cast<std::size_t>((s + (s >> 2)) / mCache.max_load_factor() + 1));
 	Log(lsDEBUG, TaggedCachePartition) << mName << " target size set to " << s;
 }
 
@@ -136,7 +136,7 @@ template<typename c_Key, typename c_Data> void TaggedCache<c_Key, c_Data>::sweep
 	int target = mLastSweep - mTargetAge;
 	int cacheRemovals = 0, mapRemovals = 0, cc = 0;
 
-	if ((mTargetSize != 0) && (mCache.size() > mTargetSize))
+	if ((mTargetSize != 0) && (static_cast<int>(mCache.size()) > mTargetSize))
 	{
 		target = mLastSweep - (mTargetAge * mTargetSize / mCache.size());
 		if (target > (mLastSweep - 2))

src/cpp/ripple/UpdateTables.cpp

@@ -0,0 +1,108 @@
+
+#include "Application.h"
+#include "Log.h"
+
+static std::vector<std::string> getSchema(DatabaseCon* dbc, const std::string& dbName)
+{
+	std::vector<std::string> schema;
+
+	std::string sql = "SELECT sql FROM sqlite_master WHERE tbl_name='";
+	sql += dbName;
+	sql += "';";
+
+	SQL_FOREACH(dbc->getDB(), sql)
+	{
+			dbc->getDB()->getStr("sql", sql);
+			schema.push_back(sql);
+	}
+
+	return schema;
+}
+
+static bool schemaHas(DatabaseCon* dbc, const std::string& dbName, int line, const std::string& content)
+{
+	std::vector<std::string> schema = getSchema(dbc, dbName);
+	if (static_cast<int>(schema.size()) <= line)
+	{
+		Log(lsFATAL) << "Schema for " << dbName << " has too few lines";
+		throw std::runtime_error("bad schema");
+	}
+	return schema[line].find(content) != std::string::npos;
+}
+
+static void addTxnSeqField()
+{
+	if (schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TxnSeq"))
+		return;
+	Log(lsWARNING) << "Transaction sequence field is missing";
+
+	Database* db = theApp->getTxnDB()->getDB();
+
+	std::vector< std::pair<uint256, int> > txIDs;
+	txIDs.reserve(300000);
+
+	Log(lsINFO) << "Parsing transactions";
+	int i = 0;
+	uint256 transID;
+	SQL_FOREACH(db, "SELECT TransID,TxnMeta FROM Transactions;")
+	{
+		std::vector<unsigned char> rawMeta;
+		int metaSize = 2048;
+		rawMeta.resize(metaSize);
+		metaSize = db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size());
+		if (metaSize > static_cast<int>(rawMeta.size()))
+		{
+			rawMeta.resize(metaSize);
+			db->getBinary("TxnMeta", &*rawMeta.begin(), rawMeta.size());
+		}
+		else rawMeta.resize(metaSize);
+
+		std::string tid;
+		db->getStr("TransID", tid);
+		transID.SetHex(tid, true);
+
+		if (rawMeta.size() == 0)
+		{
+			txIDs.push_back(std::make_pair(transID, -1));
+			Log(lsINFO) << "No metadata for " << transID;
+		}
+		else
+		{
+			TransactionMetaSet m(transID, 0, rawMeta);
+			txIDs.push_back(std::make_pair(transID, m.getIndex()));
+		}
+
+		if ((++i % 1000) == 0)
+			Log(lsINFO) << i << " transactions read";
+	}
+
+	Log(lsINFO) << "All " << i << " transactions read";
+
+	db->executeSQL("BEGIN TRANSACTION;");
+
+	Log(lsINFO) << "Dropping old index";
+	db->executeSQL("DROP INDEX AcctTxIndex;");
+
+	Log(lsINFO) << "Altering table";
+	db->executeSQL("ALTER TABLE AccountTransactions ADD COLUMN TxnSeq INTEGER;");
+
+	typedef std::pair<uint256, int> u256_int_pair_t;
+	boost::format fmt("UPDATE AccountTransactions SET TxnSeq = %d WHERE TransID = '%s';");
+	i = 0;
+	BOOST_FOREACH(u256_int_pair_t& t, txIDs)
+	{
+		db->executeSQL(boost::str(fmt % t.second % t.first.GetHex()));
+		if ((++i % 1000) == 0)
+			Log(lsINFO) << i << " transactions updated";
+	}
+
+	db->executeSQL("CREATE INDEX AcctTxIndex ON AccountTransactions(Account, LedgerSeq, TxnSeq, TransID);");
+	db->executeSQL("END TRANSACTION;");
+}
+
+void Application::updateTables()
+{ // perform any needed table updates
+	assert(schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "TransID"));
+	assert(!schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "foobar"));
+	addTxnSeqField();
+}

src/cpp/ripple/uint256.h

@@ -219,6 +219,41 @@ public:
 		return strHex(begin(), size());
 	}
 
+	void SetHexExact(const char* psz)
+	{ // must be precisely the correct number of hex digits
+		static signed char phexdigit[256] = {
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			 0, 1, 2, 3,  4, 5, 6, 7,  8, 9,-1,-1, -1,-1,-1,-1,
+
+			-1,0xa,0xb,0xc, 0xd,0xe,0xf,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,0xa,0xb,0xc, 0xd,0xe,0xf,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			-1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+			};
+
+		char* pOut	= reinterpret_cast<char*>(pn);
+		for (int i = 0; i < sizeof(pn); ++i)
+		{
+			*pOut = phexdigit[*psz++] << 4;
+			*pOut++ |= phexdigit[*psz++];
+		}
+
+		assert(*psz == 0);
+		assert(pOut == reinterpret_cast<char*>(end()));
+	}
+
 	// Allow leading whitespace.
 	// Allow leading "0x".
 	// To be valid must be '\0' terminated.
@@ -291,6 +326,11 @@ public:
 		return SetHex(str.c_str(), bStrict);
 	}
 
+	void SetHexExact(const std::string& str)
+	{
+		SetHexExact(str.c_str());
+	}
+
 	std::string ToString() const
 	{
 		return GetHex();