From ba9afb56765046379b93cc17bf78d5daca9c1277 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 17:56:30 -0700 Subject: [PATCH 01/36] Use peer add logic based on reported ledger closings. Fall back to all peers if no peers reported the ledger we want. --- src/NetworkOPs.cpp | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index ebf6284143..c711487caa 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -432,26 +432,26 @@ bool NetworkOPs::checkLastClosedLedger(const std::vector& peerLis } if (!acq->isComplete()) { // add more peers - //JED: seems like you need to do something here so it knows in beginConsensus that it isn't on the the right ledger - // switch to an empty ledger so we won't keep going - //Ledger::pointer emptyLedger(new Ledger()); - //switchLastClosedLedger(emptyLedger); - - // JED: just ask everyone + int count = 0; std::vector peers=theApp->getConnectionPool().getPeerVector(); - for(int n=0; nisConnected()) acq->peerHas(peers[n]); - } - - /* JED this wasn't adding any peers. This is also an optimization so we can do this later - // FIXME: A peer may not have a ledger just because it accepts it as the network's consensus for (std::vector::const_iterator it = peerList.begin(), end = peerList.end(); it != end; ++it) { if ((*it)->getClosedLedgerHash() == closedLedger) + { + ++count; acq->peerHas(*it); - }*/ + } + } + if (!count) + { // just ask everyone + for (std::vector::const_iterator it = peerList.begin(), end = peerList.end(); + it != end; ++it) + { + if ((*it)->isConnected()) + acq->peerHas(*it); + } + } return true; } consensus = acq->getLedger(); From 31ad073f8aa6e24a4b7838d717de3ec533663ba4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Jun 2012 17:57:02 -0700 Subject: [PATCH 02/36] Track timeouts in ledgers and transaction sets we're acquiring. --- src/LedgerAcquire.cpp | 14 +++++++++++--- src/LedgerAcquire.h | 7 +++++-- src/LedgerConsensus.cpp | 1 + 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index e440a77945..960fa1632d 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -11,8 +11,8 @@ #define LA_DEBUG #define LEDGER_ACQUIRE_TIMEOUT 2 -PeerSet::PeerSet(const uint256& hash, int interval) : mHash(hash), mTimerInterval(interval), - mComplete(false), mFailed(false), mTimer(theApp->getIOService()) +PeerSet::PeerSet(const uint256& hash, int interval) : mHash(hash), mTimerInterval(interval), mTimeouts(0), + mComplete(false), mFailed(false), mProgress(true), mTimer(theApp->getIOService()) { ; } void PeerSet::peerHas(Peer::pointer ptr) @@ -65,7 +65,12 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err { if (result == boost::asio::error::operation_aborted) return; boost::shared_ptr ptr = wptr.lock(); - if (!!ptr) ptr->onTimer(); + if (!ptr) return; + if (!ptr->mProgress) + ++ptr->mTimeouts; + else + ptr->mProgress = false; + ptr->onTimer(); } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), @@ -284,6 +289,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; + progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; mLedger->setAcquiring(); @@ -315,6 +321,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (mHaveState) mComplete = true; } trigger(peer); + progress(); return true; } @@ -345,6 +352,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (mHaveTransactions) mComplete = true; } trigger(peer); + progress(); return true; } diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 681d4bd0e0..01a0c3de06 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -20,8 +20,8 @@ class PeerSet { protected: uint256 mHash; - int mTimerInterval; - bool mComplete, mFailed; + int mTimerInterval, mTimeouts; + bool mComplete, mFailed, mProgress; boost::recursive_mutex mLock; boost::asio::deadline_timer mTimer; @@ -37,6 +37,9 @@ public: const uint256& getHash() const { return mHash; } bool isComplete() const { return mComplete; } bool isFailed() const { return mFailed; } + int getTimeouts() const { return mTimeouts; } + + void progress() { mProgress = true; } void peerHas(Peer::pointer); void badPeer(Peer::pointer); diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 5f547a821b..bc142ef2ef 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -110,6 +110,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, ++nodeDatait; } trigger(peer); + progress(); return true; } catch (...) From e7aef33344eebd0ca0b896b7905ef49ea304edfb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 16:55:26 -0700 Subject: [PATCH 03/36] Clean up timeout tracking and log acquire timeouts. --- src/LedgerAcquire.cpp | 18 +++++++++++++----- src/LedgerAcquire.h | 1 + 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 960fa1632d..825045bf7e 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -61,16 +61,24 @@ void PeerSet::resetTimer() mTimer.async_wait(boost::bind(&PeerSet::TimerEntry, pmDowncast(), boost::asio::placeholders::error)); } +void PeerSet::invokeOnTimer() +{ + if (!mProgress) + { + ++mTimeouts; + Log(lsWARNING) << "Timeout " << mTimeouts << " acquiring " << mHash.GetHex(); + } + else + mProgress = false; + onTimer(); +} + void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::error_code& result) { if (result == boost::asio::error::operation_aborted) return; boost::shared_ptr ptr = wptr.lock(); if (!ptr) return; - if (!ptr->mProgress) - ++ptr->mTimeouts; - else - ptr->mProgress = false; - ptr->onTimer(); + ptr->invokeOnTimer(); } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 01a0c3de06..87e2c86b21 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -52,6 +52,7 @@ protected: void setComplete() { mComplete = true; } void setFailed() { mFailed = true; } + void invokeOnTimer(); private: static void TimerEntry(boost::weak_ptr, const boost::system::error_code& result); From 31e9e4daab5df58b27de6e3bd10a69d366dc7a23 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 16:57:19 -0700 Subject: [PATCH 04/36] Typo. --- src/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index faaf870ae8..d51b0cd545 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -58,7 +58,7 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< sql.append(");"); std::string exists = - boost::str(boost::format("SELECT ObjType FROM CommittedObject WHERE Hash = '%s';") % hash.GetHex()); + boost::str(boost::format("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';") % hash.GetHex()); ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); Database* db = theApp->getHashNodeDB()->getDB(); From 682df845051dbfb10a00c8795a72faa77c33caf7 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 19:32:19 -0700 Subject: [PATCH 05/36] Cleanup, remove FIXMEs. --- src/Amount.cpp | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Amount.cpp b/src/Amount.cpp index 3e30ed9916..a3831035f9 100644 --- a/src/Amount.cpp +++ b/src/Amount.cpp @@ -527,14 +527,14 @@ STAmount::operator double() const } STAmount operator+(const STAmount& v1, const STAmount& v2) -{ // We can check for precision loss here (value%10)!=0 -// FIXME +{ + if (v1.isZero()) return v2; + if (v2.isZero()) return v1; + v1.throwComparable(v2); if (v1.mIsNative) return STAmount(v1.name, v1.getSNValue() + v2.getSNValue()); - if (v1.isZero()) return v2; - if (v2.isZero()) return v1; int ov1 = v1.mOffset, ov2 = v2.mOffset; int64 vv1 = static_cast(v1.mValue), vv2 = static_cast(v2.mValue); @@ -562,14 +562,12 @@ STAmount operator+(const STAmount& v1, const STAmount& v2) STAmount operator-(const STAmount& v1, const STAmount& v2) { + if (v2.isZero()) return v1; + v1.throwComparable(v2); if (v2.mIsNative) return STAmount(v1.name, v1.getSNValue() - v2.getSNValue()); - if (v2.isZero()) return v1; - if (v1.isZero() || (v2.mOffset > v1.mOffset) ) - throw std::runtime_error("value underflow"); - int ov1 = v1.mOffset, ov2 = v2.mOffset; int64 vv1 = static_cast(v1.mValue), vv2 = static_cast(v2.mValue); if (v1.mIsNegative) vv1 = -vv1; From 14795c55f4ea16614f125192d719391d14e13f94 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 19:34:25 -0700 Subject: [PATCH 06/36] Remove a FIXME or two. --- src/LedgerConsensus.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index bc142ef2ef..01487d5b68 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -427,7 +427,6 @@ int LedgerConsensus::timerEntry() theApp->getOPs().switchLastClosedLedger(consensus); mPreviousLedger = consensus; mHaveCorrectLCL = true; - // FIXME: We need some kind of idea what the consensus transaction set is } } @@ -673,7 +672,7 @@ void LedgerConsensus::applyTransaction(TransactionEngine& engine, SerializedTran if (result > 0) { Log(lsINFO) << " retry"; -// assert(!ledger->hasTransaction(txn->getTransactionID())); FIXME: We get these (doPasswordSet) + assert(!ledger->hasTransaction(txn->getTransactionID())); failedTransactions.push_back(txn); } else if (result == 0) @@ -843,9 +842,8 @@ void LedgerConsensus::accept(SHAMap::pointer set) { assert (theApp->getOPs().getNetworkTimeNC() > newLCL->getCloseTimeNC()); SerializedValidation::pointer v = boost::make_shared - (newLCLHash, newLCL->getCloseTimeNC(), mOurPosition->peekSeed(), true); + (newLCLHash, newLCL->getCloseTimeNC(), mOurPosition->peekSeed(), mProposing); v->setTrusted(); - // FIXME: If not proposing, set not full theApp->getValidations().addValidation(v); std::vector validation = v->getSigned(); newcoin::TMValidation val; From 5c41a41e4b7d924db1bc22626f142d50052f928f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 20:16:42 -0700 Subject: [PATCH 07/36] Enforce some sanity on the HashedObject code. --- src/Application.cpp | 2 +- src/Application.h | 2 ++ src/HashedObject.cpp | 69 +++++++++++++++++++++++--------------------- src/HashedObject.h | 31 ++++++++++++++------ src/SHAMap.cpp | 4 +-- 5 files changed, 63 insertions(+), 45 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index ebca2d4524..f85992d93b 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -37,7 +37,7 @@ DatabaseCon::~DatabaseCon() Application::Application() : mUNL(mIOService), - mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), + mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), mHashedObjectStore(16384, 300), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), mHashNodeDB(NULL), mNetNodeDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { diff --git a/src/Application.h b/src/Application.h index 629976dadb..4bdde4c19a 100644 --- a/src/Application.h +++ b/src/Application.h @@ -49,6 +49,7 @@ class Application NodeCache mNodeCache; ValidationCollection mValidations; SuppressionTable mSuppressions; + HashedObjectStore mHashedObjectStore; DatabaseCon *mTxnDB, *mLedgerDB, *mWalletDB, *mHashNodeDB, *mNetNodeDB; @@ -80,6 +81,7 @@ public: LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } NodeCache& getNodeCache() { return mNodeCache; } + HashedObjectStore& getHashedObjectStore() { return mHashedObjectStore; } ValidationCollection& getValidations() { return mValidations; } bool suppress(const uint256& s) { return mSuppressions.addSuppression(s); } bool suppress(const uint160& s) { return mSuppressions.addSuppression(s); } diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index d51b0cd545..63ff9f633f 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -7,7 +7,6 @@ #include "Application.h" #include "Log.h" - bool HashedObject::checkHash() const { uint256 hash = Serializer::getSHA512Half(mData); @@ -27,18 +26,19 @@ void HashedObject::setHash() mHash = Serializer::getSHA512Half(mData); } -// FIXME: Stores should be added to a queue that's services by an auxilliary thread or from an +// FIXME: Stores should be added to a queue that's serviced by an auxilliary thread or from an // auxilliary thread pool. These should be tied into a cache, since you need one to handle // an immedate read back (before the write completes) -bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector& data, - const uint256& hash) +bool HashedObjectStore::store(HashedObjectType type, uint32 index, + const std::vector& data, const uint256& hash) { if (!theApp->getHashNodeDB()) return true; -#ifdef DEBUG - Serializer s(data); - assert(hash == s.getSHA512Half()); -#endif + HashedObject::pointer object = boost::make_shared(type, index, data); + object->setHash(); + if (object->getHash() != hash) + throw std::runtime_error("Object added to store doesn't have valid hash"); + std::string sql = "INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('"; sql.append(hash.GetHex()); switch(type) @@ -51,7 +51,6 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< } sql.append(boost::lexical_cast(index)); sql.append("',"); - std::string obj; theApp->getHashNodeDB()->getDB()->escape(&(data.front()), data.size(), obj); sql.append(obj); @@ -61,22 +60,23 @@ bool HashedObject::store(HashedObjectType type, uint32 index, const std::vector< boost::str(boost::format("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';") % hash.GetHex()); ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); + if (mCache.canonicalize(hash, object)) + return false; Database* db = theApp->getHashNodeDB()->getDB(); if (SQL_EXISTS(db, exists)) return false; return db->executeSQL(sql); } -bool HashedObject::store() const +HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) { -#ifdef DEBUG - assert(checkHash()); -#endif - return store(mType, mLedgerIndex, mData, mHash); -} + HashedObject::pointer obj; + { + ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); + obj = mCache.fetch(hash); + if (obj) return obj; + } -HashedObject::pointer HashedObject::retrieve(const uint256& hash) -{ if (!theApp || !theApp->getHashNodeDB()) return HashedObject::pointer(); std::string sql = "SELECT * FROM CommittedObjects WHERE Hash='"; sql.append(hash.GetHex()); @@ -102,34 +102,37 @@ HashedObject::pointer HashedObject::retrieve(const uint256& hash) data.resize(size); db->getBinary("Object", &(data.front()), size); db->endIterRows(); - } - HashedObjectType htype = UNKNOWN; - switch(type[0]) - { - case 'L': htype = LEDGER; break; - case 'T': htype = TRANSACTION; break; - case 'A': htype = ACCOUNT_NODE; break; - case 'N': htype = TRANSACTION_NODE; break; - default: - Log(lsERROR) << "Invalid hashed object"; - return HashedObject::pointer(); - } + HashedObjectType htype = UNKNOWN; + switch(type[0]) + { + case 'L': htype = LEDGER; break; + case 'T': htype = TRANSACTION; break; + case 'A': htype = ACCOUNT_NODE; break; + case 'N': htype = TRANSACTION_NODE; break; + default: + Log(lsERROR) << "Invalid hashed object"; + return HashedObject::pointer(); + } - HashedObject::pointer obj = boost::make_shared(htype, index, data); - obj->mHash = hash; + obj = boost::make_shared(htype, index, data); + obj->mHash = hash; + mCache.canonicalize(hash, obj); + } #ifdef DEBUG assert(obj->checkHash()); #endif return obj; } -HashedObjectBulkWriter::HashedObjectBulkWriter() : sl(theApp->getHashNodeDB()->getDBLock()) +ScopedLock HashedObjectStore::beginBulk() { + ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); theApp->getHashNodeDB()->getDB()->executeSQL("BEGIN TRANSACTION;"); + return sl; } -HashedObjectBulkWriter::~HashedObjectBulkWriter() +void HashedObjectStore::endBulk() { theApp->getHashNodeDB()->getDB()->executeSQL("END TRANSACTION;"); } diff --git a/src/HashedObject.h b/src/HashedObject.h index 6a0fa28693..0f66c1509e 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -6,6 +6,7 @@ #include "types.h" #include "uint256.h" #include "ScopedLock.h" +#include "TaggedCache.h" enum HashedObjectType { @@ -34,29 +35,41 @@ public: void setHash(); const std::vector& getData() { return mData; } + const uint256& getHash() { return mHash; } +}; - bool store() const; +class HashedObjectStore +{ +protected: + TaggedCache mCache; - static bool store(HashedObjectType type, uint32 index, const std::vector& data, +public: + + HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge) { ; } + + bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); - static HashedObject::pointer retrieve(const uint256& hash); + HashedObject::pointer retrieve(const uint256& hash); + + ScopedLock beginBulk(); + void endBulk(); }; class HashedObjectBulkWriter { protected: + HashedObjectStore& mStore; ScopedLock sl; public: + HashedObjectBulkWriter(HashedObjectStore& ostore) : mStore(ostore), sl(mStore.beginBulk()) { ; } + ~HashedObjectBulkWriter() { mStore.endBulk(); } - HashedObjectBulkWriter(); - ~HashedObjectBulkWriter(); + bool store(HashedObjectType type, uint32 index, const std::vector& data, + const uint256& hash) { return mStore.store(type, index, data, hash); } - bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) - { return HashedObject::store(type, index, data, hash); } - HashedObject::pointer retrieve(const uint256& hash) - { return HashedObject::retrieve(hash); } + HashedObject::pointer retrieve(const uint256& hash) { return mStore.retrieve(hash); } }; #endif diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 987a088d31..e3961f5901 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -623,7 +623,7 @@ void SHAMapItem::dump() bool SHAMap::fetchNode(const uint256& hash, std::vector& data) { - HashedObject::pointer obj(HashedObject::retrieve(hash)); + HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); if(!obj) return false; data = obj->getData(); return true; @@ -642,7 +642,7 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) if(mDirtyNodes) { - HashedObjectBulkWriter bw; + HashedObjectBulkWriter bw(theApp->getHashedObjectStore()); boost::unordered_map& dirtyNodes = *mDirtyNodes; boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end()) From 6efa96f1eb5e28c97a307fc0d2348193977db716 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 20:28:43 -0700 Subject: [PATCH 08/36] Fix the categorization of terALREADY. --- src/TransactionEngine.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TransactionEngine.h b/src/TransactionEngine.h index 805c879546..8a78ea6da0 100644 --- a/src/TransactionEngine.h +++ b/src/TransactionEngine.h @@ -31,6 +31,7 @@ enum TransactionEngineResult tenMSG_SET, // Can't change a message key. tenBAD_AUTH_MASTER, // Auth for unclaimed account needs correct master key. tenBAD_RIPPLE, // Ledger prevents ripple from succeeding. + terALREADY, // The exact transaction was already in this ledger // Other tenFAILED = -100, // Something broke horribly @@ -43,7 +44,6 @@ enum TransactionEngineResult // terFAILED_BUT_COULD_SUCCEED = >0 // Conflict with ledger database: Fee claimed // Might succeed if not conflict is not caused by transaction ordering. - terALREADY, // The transaction was already in the ledger terBAD_SEQ, // This sequence number should be zero for prepaid transactions. terCREATED, // Can not create a previously created account. terDIR_FULL, // Can not add entry to full dir. From d2325b18ed4a82bbc4787f796bb78ce523986c66 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 20:40:29 -0700 Subject: [PATCH 09/36] cosmetic fix --- src/Peer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Peer.cpp b/src/Peer.cpp index 038d6989a9..597e25168b 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -852,7 +852,7 @@ void Peer::recvAccount(newcoin::TMAccount& packet) void Peer::recvStatus(newcoin::TMStatusChange& packet) { - Log(lsTRACE) << "Received status change from peer" << getIP(); + Log(lsTRACE) << "Received status change from peer " << getIP(); if (!packet.has_networktime()) packet.set_networktime(theApp->getOPs().getNetworkTimeNC()); mLastStatus = packet; From 09120c414b1fb4d07616512bf9728f05fb8f0851 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 20:41:51 -0700 Subject: [PATCH 10/36] Adjust timing slightly. --- src/LedgerTiming.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LedgerTiming.h b/src/LedgerTiming.h index 5a2793aa86..d7e623844d 100644 --- a/src/LedgerTiming.h +++ b/src/LedgerTiming.h @@ -19,7 +19,7 @@ # define LEDGER_CONVERGE 14 // Maximum converge time -# define LEDGER_MAX_CONVERGE 22 +# define LEDGER_MAX_CONVERGE 20 #endif From ae127674ec86e155681a9460f5faff1c91aef58e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 20:43:39 -0700 Subject: [PATCH 11/36] Remove extraneous log. --- src/LedgerConsensus.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 01487d5b68..dee196af03 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -589,10 +589,7 @@ bool LedgerConsensus::peerPosition(LedgerProposal::pointer newPosition) { assert(newPosition->getPeerID() == currentPosition->getPeerID()); if (newPosition->getProposeSeq() <= currentPosition->getProposeSeq()) - { - Log(lsINFO) << "Redundant/stale positon"; return false; - } if (newPosition->getCurrentHash() == currentPosition->getCurrentHash()) { // we missed an intermediary change Log(lsINFO) << "We missed an intermediary position"; From 08c71cbaad0aec7e5a677918714ade02d8a5d835 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 21:34:49 -0700 Subject: [PATCH 12/36] Remove debug. --- src/LedgerConsensus.cpp | 70 +++++++++++------------------------------ 1 file changed, 18 insertions(+), 52 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index dee196af03..89bdf7a48b 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -754,17 +754,6 @@ void LedgerConsensus::accept(SHAMap::pointer set) Ledger::pointer newLCL = boost::make_shared(false, boost::ref(*mPreviousLedger)); newLCL->armDirty(); -#ifdef DEBUG - Json::StyledStreamWriter ssw; - if (1) - { - Log(lsTRACE) << "newLCL before transactions"; - Json::Value p; - newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); - ssw.write(Log(lsTRACE).ref(), p); - } -#endif - CanonicalTXSet failedTransactions(set->getHash()); applyTransactions(set, newLCL, failedTransactions, true); newLCL->setClosed(); @@ -772,27 +761,23 @@ void LedgerConsensus::accept(SHAMap::pointer set) newLCL->updateHash(); uint256 newLCLHash = newLCL->getHash(); Log(lsTRACE) << "newLCL " << newLCLHash.GetHex(); - + statusChange(newcoin::neACCEPTED_LEDGER, newLCL); + if (mValidating) + { + assert (theApp->getOPs().getNetworkTimeNC() > newLCL->getCloseTimeNC()); + SerializedValidation::pointer v = boost::make_shared + (newLCLHash, newLCL->getCloseTimeNC(), mOurPosition->peekSeed(), mProposing); + v->setTrusted(); + theApp->getValidations().addValidation(v); + std::vector validation = v->getSigned(); + newcoin::TMValidation val; + val.set_validation(&validation[0], validation.size()); + theApp->getConnectionPool().relayMessage(NULL, boost::make_shared(val, newcoin::mtVALIDATION)); + Log(lsINFO) << "Validation sent " << newLCLHash.GetHex(); + } + else Log(lsWARNING) << "Not validating"; Ledger::pointer newOL = boost::make_shared(true, boost::ref(*newLCL)); - -#ifdef DEBUG - if (1) - { - Log(lsTRACE) << "newOL before transactions"; - Json::Value p; - newOL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); - ssw.write(Log(lsTRACE).ref(), p); - } - if (1) - { - Log(lsTRACE) << "current ledger"; - Json::Value p; - theApp->getMasterLedger().getCurrentLedger()->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); - ssw.write(Log(lsTRACE).ref(), p); - } -#endif - ScopedLock sl = theApp->getMasterLedger().getLock(); // Apply disputed transactions that didn't get in @@ -822,34 +807,15 @@ void LedgerConsensus::accept(SHAMap::pointer set) sl.unlock(); #ifdef DEBUG + Json::StyledStreamWriter ssw; if (1) { - Log(lsTRACE) << "newOL after current ledger transactions"; + Log(lsTRACE) << "newLCL"; Json::Value p; - newOL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); + newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); ssw.write(Log(lsTRACE).ref(), p); - Log(lsINFO) << "newLCL after transactions"; - Json::Value p2; - newLCL->addJson(p2, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); - ssw.write(Log(lsTRACE).ref(), p2); } #endif - - if (mValidating) - { - assert (theApp->getOPs().getNetworkTimeNC() > newLCL->getCloseTimeNC()); - SerializedValidation::pointer v = boost::make_shared - (newLCLHash, newLCL->getCloseTimeNC(), mOurPosition->peekSeed(), mProposing); - v->setTrusted(); - theApp->getValidations().addValidation(v); - std::vector validation = v->getSigned(); - newcoin::TMValidation val; - val.set_validation(&validation[0], validation.size()); - theApp->getConnectionPool().relayMessage(NULL, boost::make_shared(val, newcoin::mtVALIDATION)); - Log(lsINFO) << "Validation sent " << newLCLHash.GetHex(); - } - else Log(lsWARNING) << "Not validating"; - statusChange(newcoin::neACCEPTED_LEDGER, newLCL); // FIXME: If necessary, change state to TRACKING/FULL } From 4d52ec01916d100e2097353b4ae699b832a25b2e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 21:53:25 -0700 Subject: [PATCH 13/36] Don't relay stale validations. --- src/ValidationCollection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ValidationCollection.cpp b/src/ValidationCollection.cpp index a1ed90546c..5b53e46f25 100644 --- a/src/ValidationCollection.cpp +++ b/src/ValidationCollection.cpp @@ -36,7 +36,7 @@ bool ValidationCollection::addValidation(SerializedValidation::pointer val) Log(lsINFO) << "Val for " << hash.GetHex() << " from " << val->getSignerPublic().humanNodePublic() << " added " << (val->isTrusted() ? "trusted" : "UNtrusted"); - return true; + return isCurrent; } ValidationSet ValidationCollection::getValidations(const uint256& ledger) From 9db52a232a94eb1eed543f89fc7a5e613adc4fed Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 21:53:54 -0700 Subject: [PATCH 14/36] Split get signing hash operation and signature verify operation. --- src/LedgerProposal.cpp | 4 ++-- src/LedgerProposal.h | 3 ++- src/SerializedValidation.cpp | 8 ++++++-- src/SerializedValidation.h | 1 + 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/LedgerProposal.cpp b/src/LedgerProposal.cpp index 9f435415bd..01cdd9607b 100644 --- a/src/LedgerProposal.cpp +++ b/src/LedgerProposal.cpp @@ -46,9 +46,9 @@ uint256 LedgerProposal::getSigningHash() const return s.getSHA512Half(); } -bool LedgerProposal::checkSign(const std::string& signature) +bool LedgerProposal::checkSign(const std::string& signature, const uint256& signingHash) { - return mPublicKey.verifyNodePublic(getSigningHash(), signature); + return mPublicKey.verifyNodePublic(signingHash, signature); } void LedgerProposal::changePosition(const uint256& newPosition) diff --git a/src/LedgerProposal.h b/src/LedgerProposal.h index f7e50cc639..4dd32cd75d 100644 --- a/src/LedgerProposal.h +++ b/src/LedgerProposal.h @@ -36,7 +36,8 @@ public: LedgerProposal(const uint256& prevLedger, const uint256& position); uint256 getSigningHash() const; - bool checkSign(const std::string& signature); + bool checkSign(const std::string& signature, const uint256& signingHash); + bool checkSign(const std::string& signature) { return checkSign(signature, getSigningHash()); } const uint160& getPeerID() const { return mPeerID; } const uint256& getCurrentHash() const { return mCurrentHash; } diff --git a/src/SerializedValidation.cpp b/src/SerializedValidation.cpp index 9c19bc7615..b9d89e4f2a 100644 --- a/src/SerializedValidation.cpp +++ b/src/SerializedValidation.cpp @@ -56,12 +56,16 @@ uint64 SerializedValidation::getCloseTime() const } bool SerializedValidation::isValid() const +{ + return isValid(getSigningHash()); +} + +bool SerializedValidation::isValid(const uint256& signingHash) const { try { NewcoinAddress naPublicKey = NewcoinAddress::createNodePublic(getValueFieldVL(sfSigningKey)); - - return naPublicKey.isValid() && naPublicKey.verifyNodePublic(getSigningHash(), mSignature.peekValue()); + return naPublicKey.isValid() && naPublicKey.verifyNodePublic(signingHash, mSignature.peekValue()); } catch (...) { diff --git a/src/SerializedValidation.h b/src/SerializedValidation.h index ab0886a6bf..863dea31b9 100644 --- a/src/SerializedValidation.h +++ b/src/SerializedValidation.h @@ -33,6 +33,7 @@ public: bool isTrusted() const { return mTrusted; } CKey::pointer getSigningKey() const; uint256 getSigningHash() const; + bool isValid(const uint256&) const; void setTrusted() { mTrusted = true; } void addSigned(Serializer&) const; From a4b66042c59bbd28d0ef7789f35f84d6ad20bd98 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 21:54:34 -0700 Subject: [PATCH 15/36] Fix a bug where validations weren't passed to the consensus handler. Change 'suppress' to 'isNew' to better reflect the meaning of its return value. --- src/Application.h | 4 ++-- src/NetworkOPs.cpp | 29 ++++++++++++++++------------- src/Peer.cpp | 6 +++++- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/Application.h b/src/Application.h index 4bdde4c19a..b63ff58d41 100644 --- a/src/Application.h +++ b/src/Application.h @@ -83,8 +83,8 @@ public: NodeCache& getNodeCache() { return mNodeCache; } HashedObjectStore& getHashedObjectStore() { return mHashedObjectStore; } ValidationCollection& getValidations() { return mValidations; } - bool suppress(const uint256& s) { return mSuppressions.addSuppression(s); } - bool suppress(const uint160& s) { return mSuppressions.addSuppression(s); } + bool isNew(const uint256& s) { return mSuppressions.addSuppression(s); } + bool isNew(const uint160& s) { return mSuppressions.addSuppression(s); } DatabaseCon* getTxnDB() { return mTxnDB; } DatabaseCon* getLedgerDB() { return mLedgerDB; } diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index c711487caa..af6c3d2756 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -98,7 +98,7 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, } Log(lsDEBUG) << "Status other than success " << r ; - if ((mMode != omFULL) && (theApp->suppress(trans->getID()))) + if ((mMode != omFULL) && (theApp->isNew(trans->getID()))) { newcoin::TMTransaction tx; Serializer s; @@ -521,23 +521,27 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, // XXX Validate key. // XXX Take a vuc for pubkey. - NewcoinAddress naPeerPublic = NewcoinAddress::createNodePublic(strCopy(pubKey)); + NewcoinAddress naPeerPublic = NewcoinAddress::createNodePublic(strCopy(pubKey)); + LedgerProposal::pointer proposal = + boost::make_shared(mConsensus->getLCL(), proposeSeq, proposeHash, naPeerPublic); + uint256 signingHash = proposal->getSigningHash(); - if (mMode != omFULL) + if (!theApp->isNew(signingHash)) + return false; + + if ((mMode != omFULL) && (mMode != omTRACKING)) { - Log(lsINFO) << "Received proposal when not full: " << mMode; - Serializer s(signature); - return theApp->suppress(s.getSHA512Half()); + Log(lsINFO) << "Received proposal when not full/tracking: " << mMode; + return true; } + if (!mConsensus) { Log(lsWARNING) << "Received proposal when full but not during consensus window"; return false; } - LedgerProposal::pointer proposal = - boost::make_shared(mConsensus->getLCL(), proposeSeq, proposeHash, naPeerPublic); - if (!proposal->checkSign(signature)) + if (!proposal->checkSign(signature, signingHash)) { // Note that if the LCL is different, the signature check will fail Log(lsWARNING) << "Ledger proposal fails signature check"; return false; @@ -545,15 +549,14 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, // Is this node on our UNL? // XXX Is this right? - if (!theApp->getUNL().nodeInUNL(naPeerPublic)) + if (!theApp->getUNL().nodeInUNL(proposal->peekSeed())) { - Log(lsINFO) << "Relay, but no process peer proposal " << proposal->getProposeSeq() << "/" - << proposal->getCurrentHash().GetHex(); + Log(lsINFO) << "Untrusted proposal: " << naPeerPublic.humanNodePublic() << " " << + proposal->getCurrentHash().GetHex(); return true; } return mConsensus->peerPosition(proposal); - } SHAMap::pointer NetworkOPs::getTXMap(const uint256& hash) diff --git a/src/Peer.cpp b/src/Peer.cpp index 597e25168b..6159f186f0 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -730,12 +730,16 @@ void Peer::recvValidation(newcoin::TMValidation& packet) punishPeer(PP_UNKNOWN_REQUEST); return; } + try { Serializer s(packet.validation()); SerializerIterator sit(s); SerializedValidation::pointer val = boost::make_shared(boost::ref(sit)); - if (!val->isValid()) + uint256 signingHash = val->getSigningHash(); + if (!theApp->isNew(signingHash)) + return; + if (!val->isValid(signingHash)) { punishPeer(PP_UNKNOWN_REQUEST); return; From 9f519369be04d57b09aa65bd806825a09cfef67b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 21:55:45 -0700 Subject: [PATCH 16/36] Don't relay failed transactions when we are tracking the network. --- src/NetworkOPs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index af6c3d2756..7931e0bf2c 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -98,7 +98,7 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, } Log(lsDEBUG) << "Status other than success " << r ; - if ((mMode != omFULL) && (theApp->isNew(trans->getID()))) + if ((mMode != omFULL) && (mMode != omTRACKING) && (theApp->isNew(trans->getID()))) { newcoin::TMTransaction tx; Serializer s; From c70f66b286da1c1287e82d3265d360312e2de58b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 22:16:33 -0700 Subject: [PATCH 17/36] Fix a bug I just introduced. --- src/NetworkOPs.cpp | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 7931e0bf2c..786641ceb5 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -521,12 +521,13 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, // XXX Validate key. // XXX Take a vuc for pubkey. - NewcoinAddress naPeerPublic = NewcoinAddress::createNodePublic(strCopy(pubKey)); - LedgerProposal::pointer proposal = - boost::make_shared(mConsensus->getLCL(), proposeSeq, proposeHash, naPeerPublic); - uint256 signingHash = proposal->getSigningHash(); - if (!theApp->isNew(signingHash)) + // Get a preliminary hash to use to suppress duplicates + Serializer s; + s.add32(proposeSeq); + s.add32(getCurrentLedgerID()); + s.addRaw(pubKey); + if (!theApp->isNew(s.getSHA512Half())) return false; if ((mMode != omFULL) && (mMode != omTRACKING)) @@ -541,7 +542,10 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, return false; } - if (!proposal->checkSign(signature, signingHash)) + NewcoinAddress naPeerPublic = NewcoinAddress::createNodePublic(strCopy(pubKey)); + LedgerProposal::pointer proposal = + boost::make_shared(mConsensus->getLCL(), proposeSeq, proposeHash, naPeerPublic); + if (!proposal->checkSign(signature)) { // Note that if the LCL is different, the signature check will fail Log(lsWARNING) << "Ledger proposal fails signature check"; return false; From 8e1c49f0d70050e4d9b67b7697caac83b9945998 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 22:28:49 -0700 Subject: [PATCH 18/36] Bugfixes. --- src/LedgerConsensus.cpp | 2 +- src/LedgerProposal.cpp | 1 - src/LedgerProposal.h | 12 ++++++------ src/NetworkOPs.cpp | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 89bdf7a48b..5baeff0d6a 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -766,7 +766,7 @@ void LedgerConsensus::accept(SHAMap::pointer set) { assert (theApp->getOPs().getNetworkTimeNC() > newLCL->getCloseTimeNC()); SerializedValidation::pointer v = boost::make_shared - (newLCLHash, newLCL->getCloseTimeNC(), mOurPosition->peekSeed(), mProposing); + (newLCLHash, newLCL->getCloseTimeNC(), mValSeed, mProposing); v->setTrusted(); theApp->getValidations().addValidation(v); std::vector validation = v->getSigned(); diff --git a/src/LedgerProposal.cpp b/src/LedgerProposal.cpp index 01cdd9607b..7d79bf0b6b 100644 --- a/src/LedgerProposal.cpp +++ b/src/LedgerProposal.cpp @@ -22,7 +22,6 @@ LedgerProposal::LedgerProposal(const uint256& pLgr, uint32 seq, const uint256& t LedgerProposal::LedgerProposal(const NewcoinAddress& naSeed, const uint256& prevLgr, const uint256& position) : mPreviousLedger(prevLgr), mCurrentHash(position), mProposeSeq(0) { - mSeed = naSeed; mPublicKey = NewcoinAddress::createNodePublic(naSeed); mPrivateKey = NewcoinAddress::createNodePrivate(naSeed); mPeerID = mPublicKey.getNodeID(); diff --git a/src/LedgerProposal.h b/src/LedgerProposal.h index 4dd32cd75d..35557920c5 100644 --- a/src/LedgerProposal.h +++ b/src/LedgerProposal.h @@ -16,9 +16,8 @@ protected: uint32 mProposeSeq; uint160 mPeerID; - NewcoinAddress mPublicKey; // Peer - NewcoinAddress mPrivateKey; // Our's - NewcoinAddress mSeed; // Our's + NewcoinAddress mPublicKey; + NewcoinAddress mPrivateKey; // If ours static const uint32 sProposeMagic = 0x50525000; // PRP @@ -27,10 +26,11 @@ public: typedef boost::shared_ptr pointer; // proposal from peer - LedgerProposal(const uint256& prevLgr, uint32 proposeSeq, const uint256& propose, const NewcoinAddress& naPeerPublic); + LedgerProposal(const uint256& prevLgr, uint32 proposeSeq, const uint256& propose, + const NewcoinAddress& naPeerPublic); // our first proposal - LedgerProposal(const NewcoinAddress& naSeed, const uint256& prevLedger, const uint256& position); + LedgerProposal(const NewcoinAddress& privKey, const uint256& prevLedger, const uint256& position); // an unsigned proposal LedgerProposal(const uint256& prevLedger, const uint256& position); @@ -43,7 +43,7 @@ public: const uint256& getCurrentHash() const { return mCurrentHash; } const uint256& getPrevLedger() const { return mPreviousLedger; } uint32 getProposeSeq() const { return mProposeSeq; } - const NewcoinAddress& peekSeed() const { return mSeed; } + const NewcoinAddress& peekPublic() const { return mPublicKey; } std::vector getPubKey() const { return mPublicKey.getNodePublic(); } std::vector sign(); diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 786641ceb5..9280ea83d2 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -553,7 +553,7 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, // Is this node on our UNL? // XXX Is this right? - if (!theApp->getUNL().nodeInUNL(proposal->peekSeed())) + if (!theApp->getUNL().nodeInUNL(proposal->peekPublic())) { Log(lsINFO) << "Untrusted proposal: " << naPeerPublic.humanNodePublic() << " " << proposal->getCurrentHash().GetHex(); From 5073185354cbaf812b8063a27e68af267026a062 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 22:45:24 -0700 Subject: [PATCH 19/36] Remove a redundant and expensive test. --- src/NetworkOPs.cpp | 1 - src/Peer.cpp | 5 ++++- src/SerializedValidation.cpp | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 9280ea83d2..e4d8b4af42 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -647,7 +647,6 @@ std::vector< std::pair > bool NetworkOPs::recvValidation(SerializedValidation::pointer val) { Log(lsINFO) << "recvValidation " << val->getLedgerHash().GetHex(); - return theApp->getValidations().addValidation(val); } diff --git a/src/Peer.cpp b/src/Peer.cpp index 6159f186f0..f4cbbcb0a9 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -735,15 +735,18 @@ void Peer::recvValidation(newcoin::TMValidation& packet) { Serializer s(packet.validation()); SerializerIterator sit(s); - SerializedValidation::pointer val = boost::make_shared(boost::ref(sit)); + SerializedValidation::pointer val = boost::make_shared(boost::ref(sit), false); + uint256 signingHash = val->getSigningHash(); if (!theApp->isNew(signingHash)) return; + if (!val->isValid(signingHash)) { punishPeer(PP_UNKNOWN_REQUEST); return; } + if (theApp->getOPs().recvValidation(val)) { PackedMessage::pointer message = boost::make_shared(packet, newcoin::mtVALIDATION); diff --git a/src/SerializedValidation.cpp b/src/SerializedValidation.cpp index b9d89e4f2a..71c11c831c 100644 --- a/src/SerializedValidation.cpp +++ b/src/SerializedValidation.cpp @@ -16,7 +16,7 @@ const uint32 SerializedValidation::sValidationMagic = 0x4c575200; // "LGR" SerializedValidation::SerializedValidation(SerializerIterator& sit, bool checkSignature) : STObject(sValidationFormat, sit), mSignature(sit, "Signature"), mTrusted(false) { - if (!isValid()) throw std::runtime_error("Invalid validation"); + if (checkSignature && !isValid()) throw std::runtime_error("Invalid validation"); } SerializedValidation::SerializedValidation(const uint256& ledgerHash, uint64 closeTime, From 726b7dc5720cd3360de4110f25cb8c99bdf69967 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 23 Jun 2012 22:55:53 -0700 Subject: [PATCH 20/36] Speed up DB initialization --- src/Wallet.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Wallet.cpp b/src/Wallet.cpp index 9f12140f0f..12dda320b6 100644 --- a/src/Wallet.cpp +++ b/src/Wallet.cpp @@ -76,7 +76,21 @@ bool Wallet::nodeIdentityCreate() { // Make new key. +#ifdef CREATE_NEW_DH_PARAMS std::string strDh512 = DH_der_gen(512); +#else + static char dh512Param[] = { + 0x30, 0x46, 0x02, 0x41, 0x00, 0x98, 0x15, 0xd2, 0xd0, 0x08, 0x32, 0xda, + 0xaa, 0xac, 0xc4, 0x71, 0xa3, 0x1b, 0x11, 0xf0, 0x6c, 0x62, 0xb2, 0x35, + 0x8a, 0x10, 0x92, 0xc6, 0x0a, 0xa3, 0x84, 0x7e, 0xaf, 0x17, 0x29, 0x0b, + 0x70, 0xef, 0x07, 0x4f, 0xfc, 0x9d, 0x6d, 0x87, 0x99, 0x19, 0x09, 0x5b, + 0x6e, 0xdb, 0x57, 0x72, 0x4a, 0x7e, 0xcd, 0xaf, 0xbd, 0x3a, 0x97, 0x55, + 0x51, 0x77, 0x5a, 0x34, 0x7c, 0xe8, 0xc5, 0x71, 0x63, 0x02, 0x01, 0x02 + }; + std::string strDh512(dh512Param, sizeof(dh512Param)); +#endif + + #if 1 std::string strDh1024 = strDh512; // For testing and most cases 512 is fine. #else From 4a15c2a105035ffcdbf804254a9caa1e8779dbad Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 04:52:01 -0700 Subject: [PATCH 21/36] Bugfix. --- src/TaggedCache.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index a53b7d3dc5..8c84a34654 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -121,7 +121,7 @@ template bool TaggedCache::touch // Is the object in the map? typename boost::unordered_map::iterator mit = mMap.find(key); if (mit == mMap.end()) return false; - if (mit->second->expired()) + if (mit->second.expired()) { // in map, but expired mMap.erase(mit); return false; @@ -131,12 +131,12 @@ template bool TaggedCache::touch typename boost::unordered_map::iterator cit = mCache.find(key); if (cit != mCache.end()) { // in both map and cache - cit->second->first = time(NULL); + cit->second.first = time(NULL); return true; } // In map but not cache, put in cache - mCache.insert(std::make_pair(key, std::make_pair(time(NULL), weak_data_ptr(cit->second->second)))); + mCache.insert(std::make_pair(key, std::make_pair(time(NULL), weak_data_ptr(cit->second.second)))); return true; } From 748461ddc22df698076b60d48b37104357b58890 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 04:53:31 -0700 Subject: [PATCH 22/36] Make the hashed object write code not block on an SQL write. Make bulk updates efficient. --- src/Application.cpp | 1 + src/HashedObject.cpp | 107 +++++++++++++++++++++++++++---------------- src/HashedObject.h | 31 ++++--------- src/SHAMap.cpp | 3 +- 4 files changed, 80 insertions(+), 62 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index f85992d93b..e500b39eeb 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -51,6 +51,7 @@ extern int TxnDBCount, LedgerDBCount, WalletDBCount, HashNodeDBCount, NetNodeDBC void Application::stop() { mIOService.stop(); + mHashedObjectStore.bulkWrite(); Log(lsINFO) << "Stopped: " << mIOService.stopped(); } diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 63ff9f633f..03d885b71e 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -26,46 +26,79 @@ void HashedObject::setHash() mHash = Serializer::getSHA512Half(mData); } -// FIXME: Stores should be added to a queue that's serviced by an auxilliary thread or from an -// auxilliary thread pool. These should be tied into a cache, since you need one to handle -// an immedate read back (before the write completes) +HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : + mCache(cacheSize, cacheAge), mWritePending(false) +{ + mWriteSet.reserve(128); +} + bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) -{ +{ // return: false=already in cache, true = added to cache if (!theApp->getHashNodeDB()) return true; + if (mCache.touch(hash)) return false; + HashedObject::pointer object = boost::make_shared(type, index, data); object->setHash(); if (object->getHash() != hash) throw std::runtime_error("Object added to store doesn't have valid hash"); - std::string sql = "INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('"; - sql.append(hash.GetHex()); - switch(type) + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + mWriteSet.push_back(object); + if (mWriteSet.size() == 64) { - case LEDGER: sql.append("','L','"); break; - case TRANSACTION: sql.append("','T','"); break; - case ACCOUNT_NODE: sql.append("','A','"); break; - case TRANSACTION_NODE: sql.append("','N','"); break; - default: sql.append("','U','"); break; + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + if (!mWritePending) + { + mWritePending = true; + boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this)); + t.detach(); + } } - sql.append(boost::lexical_cast(index)); - sql.append("',"); - std::string obj; - theApp->getHashNodeDB()->getDB()->escape(&(data.front()), data.size(), obj); - sql.append(obj); - sql.append(");"); + return true; +} - std::string exists = - boost::str(boost::format("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';") % hash.GetHex()); +void HashedObjectStore::bulkWrite() +{ + std::vector< boost::shared_ptr > set; + set.reserve(128); + + { + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + mWriteSet.swap(set); + mWritePending = false; + } + + boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); + boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); - ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); - if (mCache.canonicalize(hash, object)) - return false; Database* db = theApp->getHashNodeDB()->getDB(); - if (SQL_EXISTS(db, exists)) - return false; - return db->executeSQL(sql); + ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); + + db->executeSQL("BEGIN TRANSACTION;"); + + for (std::vector< boost::shared_ptr >::iterator it = set.begin(), end = set.end(); it != end; ++it) + { + HashedObject& obj = **it; + if (!SQL_EXISTS(db, boost::str(fExists % obj.getHash().GetHex()))) + { + char type; + switch(obj.getType()) + { + case LEDGER: type = 'L'; break; + case TRANSACTION: type = 'T'; break; + case ACCOUNT_NODE: type = 'A'; break; + case TRANSACTION_NODE: type = 'N'; break; + default: type = 'U'; + } + std::string rawData; + db->escape(&(obj.getData().front()), obj.getData().size(), rawData); + db->executeSQL(boost::str(fAdd % obj.getHash().GetHex() % type % obj.getIndex() % rawData )); + } + } + + db->executeSQL("END TRANSACTION;"); } HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) @@ -74,7 +107,11 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) { ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); obj = mCache.fetch(hash); - if (obj) return obj; + if (obj) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: incache"; + return obj; + } } if (!theApp || !theApp->getHashNodeDB()) return HashedObject::pointer(); @@ -90,7 +127,10 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) Database* db = theApp->getHashNodeDB()->getDB(); if (!db->executeSQL(sql) || !db->startIterRows()) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: not in db"; return HashedObject::pointer(); + } std::string type; db->getStr("ObjType", type); @@ -122,19 +162,8 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) #ifdef DEBUG assert(obj->checkHash()); #endif + Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: in db"; return obj; } -ScopedLock HashedObjectStore::beginBulk() -{ - ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); - theApp->getHashNodeDB()->getDB()->executeSQL("BEGIN TRANSACTION;"); - return sl; -} - -void HashedObjectStore::endBulk() -{ - theApp->getHashNodeDB()->getDB()->executeSQL("END TRANSACTION;"); -} - // vim:ts=4 diff --git a/src/HashedObject.h b/src/HashedObject.h index 0f66c1509e..514a026670 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -34,8 +34,10 @@ public: bool checkFixHash(); void setHash(); - const std::vector& getData() { return mData; } - const uint256& getHash() { return mHash; } + const std::vector& getData() { return mData; } + const uint256& getHash() { return mHash; } + HashedObjectType getType() { return mType; } + uint32 getIndex() { return mLedgerIndex; } }; class HashedObjectStore @@ -43,33 +45,20 @@ class HashedObjectStore protected: TaggedCache mCache; + boost::recursive_mutex mWriteMutex; + std::vector< boost::shared_ptr > mWriteSet; + bool mWritePending; + public: - HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge) { ; } + HashedObjectStore(int cacheSize, int cacheAge); bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); HashedObject::pointer retrieve(const uint256& hash); - ScopedLock beginBulk(); - void endBulk(); -}; - -class HashedObjectBulkWriter -{ -protected: - HashedObjectStore& mStore; - ScopedLock sl; - -public: - HashedObjectBulkWriter(HashedObjectStore& ostore) : mStore(ostore), sl(mStore.beginBulk()) { ; } - ~HashedObjectBulkWriter() { mStore.endBulk(); } - - bool store(HashedObjectType type, uint32 index, const std::vector& data, - const uint256& hash) { return mStore.store(type, index, data, hash); } - - HashedObject::pointer retrieve(const uint256& hash) { return mStore.retrieve(hash); } + void bulkWrite(); }; #endif diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index e3961f5901..ea592d60a5 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -642,14 +642,13 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) if(mDirtyNodes) { - HashedObjectBulkWriter bw(theApp->getHashedObjectStore()); boost::unordered_map& dirtyNodes = *mDirtyNodes; boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end()) { s.erase(); it->second->addRaw(s); - bw.store(t, seq, s.peekData(), s.getSHA512Half()); + theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half()); if (flushed++ >= maxNodes) return flushed; it = dirtyNodes.erase(it); From 955752c8bc9ef57d881af77d147caf49b7a40b1f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:33:32 -0700 Subject: [PATCH 23/36] Bugfix. --- src/TaggedCache.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index 8c84a34654..f07a80d138 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -213,9 +213,8 @@ boost::shared_ptr TaggedCache::fetch(const key_type& key) template bool TaggedCache::store(const key_type& key, const c_Data& data) { - if (!canonicalize(key, boost::shared_ptr(data))) - return false; - return true; + boost::shared_ptr d = boost::make_shared(boost::ref(data)); + return canonicalize(key, d); } template From c13c561295a2efdc54687f91ed9ce76f3f64ac92 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:33:53 -0700 Subject: [PATCH 24/36] Use the new accelerators to speed up ledger and transaction set synchronization --- src/LedgerAcquire.cpp | 16 ++++++--- src/LedgerAcquire.h | 26 -------------- src/LedgerConsensus.cpp | 13 +++---- src/LedgerConsensus.h | 1 - src/SHAMapSync.h | 75 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 38 deletions(-) create mode 100644 src/SHAMapSync.h diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 825045bf7e..9801719275 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -7,6 +7,7 @@ #include "Application.h" #include "Log.h" +#include "SHAMapSync.h" #define LA_DEBUG #define LEDGER_ACQUIRE_TIMEOUT 2 @@ -82,7 +83,7 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), - mFilter(&theApp->getNodeCache()), mHaveBase(false), mHaveState(false), mHaveTransactions(false) + mHaveBase(false), mHaveState(false), mHaveTransactions(false) { #ifdef LA_DEBUG Log(lsTRACE) << "Acquiring ledger " << mHash.GetHex(); @@ -168,7 +169,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &tFilter); if (nodeIDs.empty()) { if (!mLedger->peekTransactionMap()->isValid()) mFailed = true; @@ -220,7 +222,8 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &mFilter); + AccountStateSF aFilter(mLedger->getHash(), mLedger->getSeq()); + mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &aFilter); if (nodeIDs.empty()) { if (!mLedger->peekAccountStateMap()->isValid()) mFailed = true; @@ -297,6 +300,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), data, mHash); progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; @@ -311,6 +315,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -318,7 +323,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait)) return false; } - else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; @@ -342,6 +347,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + AccountStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -349,7 +355,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait)) return false; } - else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 87e2c86b21..cc499fdb3a 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -58,31 +58,6 @@ private: static void TimerEntry(boost::weak_ptr, const boost::system::error_code& result); }; -typedef TaggedCache< uint256, std::vector > NodeCache; -typedef std::vector VUC; - -class THSyncFilter : public SHAMapSyncFilter -{ -protected: - NodeCache* mCache; // holds nodes we see during the consensus process - -public: - THSyncFilter(NodeCache* cache) : mCache(cache) { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, - const std::vector& nodeData, bool) - { - boost::shared_ptr ptr = boost::make_shared(nodeData); - mCache->canonicalize(nodeHash, ptr); - } - virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - boost::shared_ptr entry = mCache->fetch(nodeHash); - if (!entry) return false; - nodeData = *entry; - return true; - } -}; - class LedgerAcquire : public PeerSet, public boost::enable_shared_from_this { // A ledger we are trying to acquire public: @@ -90,7 +65,6 @@ public: protected: Ledger::pointer mLedger; - THSyncFilter mFilter; bool mHaveBase, mHaveState, mHaveTransactions; std::vector< boost::function > mOnComplete; diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 5baeff0d6a..ce5a35f2b8 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -11,13 +11,13 @@ #include "LedgerTiming.h" #include "SerializedValidation.h" #include "Log.h" +#include "SHAMapSync.h" #define TRUST_NETWORK // #define LC_DEBUG -TransactionAcquire::TransactionAcquire(const uint256& hash) - : PeerSet(hash, 1), mFilter(&theApp->getNodeCache()), mHaveRoot(false) +TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, 1), mHaveRoot(false) { mMap = boost::make_shared(); mMap->setSynching(); @@ -50,9 +50,9 @@ void TransactionAcquire::trigger(Peer::pointer peer) } if (mHaveRoot) { - std::vector nodeIDs; - std::vector nodeHashes; - mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &mFilter); + std::vector nodeIDs; std::vector nodeHashes; + ConsensusTransSetSF sf; + mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &sf); if (nodeIDs.empty()) { if (mMap->isValid()) @@ -91,6 +91,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, { std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); + ConsensusTransSetSF sf; while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -104,7 +105,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, return false; else mHaveRoot = true; } - else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &mFilter)) + else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) return false; ++nodeIDit; ++nodeDatait; diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index c9a5010cee..8db236487e 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -22,7 +22,6 @@ public: protected: SHAMap::pointer mMap; - THSyncFilter mFilter; // FIXME: Should use transaction master too bool mHaveRoot; void onTimer() { trigger(Peer::pointer()); } diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h new file mode 100644 index 0000000000..d88b98fd51 --- /dev/null +++ b/src/SHAMapSync.h @@ -0,0 +1,75 @@ +#ifndef __SHAMAPYSNC__ +#define __SHAMAPSYNC__ + +#include "SHAMap.h" +#include "Application.h" + +// Sync filters allow low-level SHAMapSync code to interact correctly with +// higher-level structures such as caches and transaction stores + +class ConsensusTransSetSF : public SHAMapSyncFilter +{ // sync filter for transaction sets during consensus building +public: + ConsensusTransSetSF() { ; } + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + // WRITEME: If 'isLeaf' is true, this is a transaction + theApp->getNodeCache().store(nodeHash, nodeData); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + // WRITEME: We could check our own map, we could check transaction tables + return theApp->getNodeCache().retrieve(nodeHash, nodeData); + } +}; + +class LedgerAccountStateSF : public SHAMapSyncFilter +{ // sync filter for account state nodes during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + LedgerAccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +class TransactionStateSF : public SHAMapSyncFilter +{ // sync filter for transactions tree during ledger sync +protected: + uint256 mLedgerHash; + uint32 mLedgerSeq; + +public: + TransactionStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + { ; } + + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, bool isLeaf) + { + theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); + } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) + { + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); + if (!node) return false; + nodeData = node->getData(); + return true; + } +}; + +#endif From f9560e27c75230dab1147562bb1cb9c91a0faafc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:42:44 -0700 Subject: [PATCH 25/36] Missing #include --- src/TaggedCache.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/TaggedCache.h b/src/TaggedCache.h index f07a80d138..bb5dc36667 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -6,6 +6,7 @@ #include #include #include +#include // This class implemented a cache and a map. The cache keeps objects alive // in the map. The map allows multiple code paths that reference objects From 340f17e471ed6b160b3c7ad857298685e5741038 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 05:48:09 -0700 Subject: [PATCH 26/36] Use the new filters. --- src/HashedObject.cpp | 14 +++++++++----- src/LedgerAcquire.cpp | 10 +++++----- src/SHAMapSync.h | 4 ++-- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 03d885b71e..6d59e8b293 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -37,25 +37,28 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false=already in cache, true = added to cache if (!theApp->getHashNodeDB()) return true; - if (mCache.touch(hash)) return false; + if (mCache.touch(hash)) + { + Log(lsTRACE) << "HOS: " << hash.GetHex() << " store: incache"; + return false; + } HashedObject::pointer object = boost::make_shared(type, index, data); object->setHash(); if (object->getHash() != hash) throw std::runtime_error("Object added to store doesn't have valid hash"); - boost::recursive_mutex::scoped_lock sl(mWriteMutex); - mWriteSet.push_back(object); - if (mWriteSet.size() == 64) { boost::recursive_mutex::scoped_lock sl(mWriteMutex); - if (!mWritePending) + mWriteSet.push_back(object); + if (!mWritePending && (mWriteSet.size() >= 64)) { mWritePending = true; boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this)); t.detach(); } } + Log(lsTRACE) << "HOS: " << hash.GetHex() << " store: deferred"; return true; } @@ -69,6 +72,7 @@ void HashedObjectStore::bulkWrite() mWriteSet.swap(set); mWritePending = false; } + Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 9801719275..c87d16ee3a 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -169,7 +169,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); mLedger->peekTransactionMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &tFilter); if (nodeIDs.empty()) { @@ -222,7 +222,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) { std::vector nodeIDs; std::vector nodeHashes; - AccountStateSF aFilter(mLedger->getHash(), mLedger->getSeq()); + AccountStateSF aFilter(mLedger->getHash(), mLedger->getLedgerSeq()); mLedger->peekAccountStateMap()->getMissingNodes(nodeIDs, nodeHashes, 128, &aFilter); if (nodeIDs.empty()) { @@ -300,7 +300,7 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; - theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), data, mHash); + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), strCopy(data), mHash); progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; @@ -315,7 +315,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); - TransactionStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + TransactionStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) @@ -347,7 +347,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); std::list< std::vector >::const_iterator nodeDatait = data.begin(); - AccountStateSF tFilter(mLedger->getHash(), mLedger->getSeq()); + AccountStateSF tFilter(mLedger->getHash(), mLedger->getLedgerSeq()); while (nodeIDit != nodeIDs.end()) { if (nodeIDit->isRoot()) diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h index d88b98fd51..26b6185520 100644 --- a/src/SHAMapSync.h +++ b/src/SHAMapSync.h @@ -24,14 +24,14 @@ public: } }; -class LedgerAccountStateSF : public SHAMapSyncFilter +class AccountStateSF : public SHAMapSyncFilter { // sync filter for account state nodes during ledger sync protected: uint256 mLedgerHash; uint32 mLedgerSeq; public: - LedgerAccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) + AccountStateSF(const uint256& ledgerHash, uint32 ledgerSeq) : mLedgerHash(ledgerHash), mLedgerSeq(ledgerSeq) { ; } virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, From 08bfd8f1d04e66ca8aca48963fb39a4eb841670f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:01:48 -0700 Subject: [PATCH 27/36] Fix the disconnect between hashes of nodes in transport format and hashes of nodes in internal format. --- src/HashedObject.cpp | 30 ++---------------------------- src/HashedObject.h | 8 ++------ 2 files changed, 4 insertions(+), 34 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 6d59e8b293..512a0ab025 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -7,25 +7,6 @@ #include "Application.h" #include "Log.h" -bool HashedObject::checkHash() const -{ - uint256 hash = Serializer::getSHA512Half(mData); - return hash == mHash; -} - -bool HashedObject::checkFixHash() -{ - uint256 hash = Serializer::getSHA512Half(mData); - if (hash == mHash) return true; - mHash = hash; - return false; -} - -void HashedObject::setHash() -{ - mHash = Serializer::getSHA512Half(mData); -} - HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : mCache(cacheSize, cacheAge), mWritePending(false) { @@ -43,10 +24,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, return false; } - HashedObject::pointer object = boost::make_shared(type, index, data); - object->setHash(); - if (object->getHash() != hash) - throw std::runtime_error("Object added to store doesn't have valid hash"); + HashedObject::pointer object = boost::make_shared(type, index, data, hash); { boost::recursive_mutex::scoped_lock sl(mWriteMutex); @@ -159,13 +137,9 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) return HashedObject::pointer(); } - obj = boost::make_shared(htype, index, data); - obj->mHash = hash; + obj = boost::make_shared(htype, index, data, hash); mCache.canonicalize(hash, obj); } -#ifdef DEBUG - assert(obj->checkHash()); -#endif Log(lsTRACE) << "HOS: " << hash.GetHex() << " fetch: in db"; return obj; } diff --git a/src/HashedObject.h b/src/HashedObject.h index 514a026670..0331cfa9b3 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -27,12 +27,8 @@ public: uint32 mLedgerIndex; std::vector mData; - HashedObject(HashedObjectType type, uint32 index, const std::vector& data) : - mType(type), mLedgerIndex(index), mData(data) { ; } - - bool checkHash() const; - bool checkFixHash(); - void setHash(); + HashedObject(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) : + mType(type), mHash(hash), mLedgerIndex(index), mData(data) { ; } const std::vector& getData() { return mData; } const uint256& getHash() { return mHash; } From 9d5467feffd6780d71fe6ecbd807a47f417b78d2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:15:19 -0700 Subject: [PATCH 28/36] Bugfix. --- src/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 512a0ab025..dd62f2594c 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -53,7 +53,7 @@ void HashedObjectStore::bulkWrite() Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); - boost::format fAdd("INSERT INTO ComittedObject (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u','%s');"); + boost::format fAdd("INSERT INTO ComittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); Database* db = theApp->getHashNodeDB()->getDB(); ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); From 29586c8e7d5956c78b1f16845cdbd229bb42d689 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:16:32 -0700 Subject: [PATCH 29/36] Typo. --- src/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index dd62f2594c..36f4273bc2 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -53,7 +53,7 @@ void HashedObjectStore::bulkWrite() Log(lsINFO) << "HOS: BulkWrite " << set.size(); boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); - boost::format fAdd("INSERT INTO ComittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); + boost::format fAdd("INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); Database* db = theApp->getHashNodeDB()->getDB(); ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); From 324992b2ad99494655d4e9065b60720cb1c9ad85 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 06:31:03 -0700 Subject: [PATCH 30/36] Bugfixes. --- src/Application.cpp | 2 +- src/Application.h | 4 ++-- src/Ledger.cpp | 15 +++++++++++---- src/LedgerHistory.cpp | 8 +++++--- src/SHAMapSync.h | 18 ++++++------------ 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index e500b39eeb..6b7ff795a2 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -37,7 +37,7 @@ DatabaseCon::~DatabaseCon() Application::Application() : mUNL(mIOService), - mNetOps(mIOService, &mMasterLedger), mNodeCache(16384, 600), mHashedObjectStore(16384, 300), + mNetOps(mIOService, &mMasterLedger), mTempNodeCache(16384, 90), mHashedObjectStore(16384, 300), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), mHashNodeDB(NULL), mNetNodeDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL) { diff --git a/src/Application.h b/src/Application.h index b63ff58d41..ff0024adc7 100644 --- a/src/Application.h +++ b/src/Application.h @@ -46,7 +46,7 @@ class Application LedgerAcquireMaster mMasterLedgerAcquire; TransactionMaster mMasterTransaction; NetworkOPs mNetOps; - NodeCache mNodeCache; + NodeCache mTempNodeCache; ValidationCollection mValidations; SuppressionTable mSuppressions; HashedObjectStore mHashedObjectStore; @@ -80,7 +80,7 @@ public: LedgerMaster& getMasterLedger() { return mMasterLedger; } LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } TransactionMaster& getMasterTransaction() { return mMasterTransaction; } - NodeCache& getNodeCache() { return mNodeCache; } + NodeCache& getTempNodeCache() { return mTempNodeCache; } HashedObjectStore& getHashedObjectStore() { return mHashedObjectStore; } ValidationCollection& getValidations() { return mValidations; } bool isNew(const uint256& s) { return mSuppressions.addSuppression(s); } diff --git a/src/Ledger.cpp b/src/Ledger.cpp index e583647a6a..1ab8fb3d5c 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -5,6 +5,8 @@ #include #include +#include "../json/writer.h" + #include "Application.h" #include "Ledger.h" #include "utils.h" @@ -350,6 +352,11 @@ Ledger::pointer Ledger::getSQL(const std::string& sql) boost::make_shared(prevHash, transHash, accountHash, totCoins, closingTime, ledgerSeq); if (ret->getHash() != ledgerHash) { + Json::StyledStreamWriter ssw; + Log(lsERROR) << "Failed on ledger"; + Json::Value p; + ret->addJson(p, LEDGER_JSON_FULL); + ssw.write(Log(lsERROR).ref(), p); assert(false); return Ledger::pointer(); } @@ -379,7 +386,8 @@ void Ledger::addJson(Json::Value& ret, int options) boost::recursive_mutex::scoped_lock sl(mLock); ledger["parentHash"] = mParentHash.GetHex(); - if(mClosed) + bool full = (options & LEDGER_JSON_FULL) != 0; + if(mClosed || full) { ledger["hash"] = mHash.GetHex(); ledger["transactionHash"] = mTransHash.GetHex(); @@ -391,8 +399,7 @@ void Ledger::addJson(Json::Value& ret, int options) else ledger["closed"] = false; if (mCloseTime != 0) ledger["closeTime"] = boost::posix_time::to_simple_string(ptFromSeconds(mCloseTime)); - bool full = (options & LEDGER_JSON_FULL) != 0; - if (full || ((options & LEDGER_JSON_DUMP_TXNS) != 0)) + if (mTransactionMap && (full || ((options & LEDGER_JSON_DUMP_TXNS) != 0))) { Json::Value txns(Json::arrayValue); for (SHAMapItem::pointer item = mTransactionMap->peekFirstItem(); !!item; @@ -408,7 +415,7 @@ void Ledger::addJson(Json::Value& ret, int options) } ledger["transactions"] = txns; } - if (full || ((options & LEDGER_JSON_DUMP_STATE) != 0)) + if (mAccountStateMap && (full || ((options & LEDGER_JSON_DUMP_STATE) != 0))) { Json::Value state(Json::arrayValue); for (SHAMapItem::pointer item = mAccountStateMap->peekFirstItem(); !!item; diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index f20a7ae53a..e42e13020e 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -60,9 +60,11 @@ Ledger::pointer LedgerHistory::getLedgerByHash(const uint256& hash) Ledger::pointer ret = mLedgersByHash.fetch(hash); if (ret) return ret; - ret = Ledger::loadByHash(hash); - if (!ret) return ret; - assert(ret->getHash() == hash); +// FIXME: A ledger without SHA maps isn't very useful +// This code will need to build them +// ret = Ledger::loadByHash(hash); +// if (!ret) return ret; +// assert(ret->getHash() == hash); boost::recursive_mutex::scoped_lock sl(mLedgersByHash.peekMutex()); mLedgersByHash.canonicalize(hash, ret); diff --git a/src/SHAMapSync.h b/src/SHAMapSync.h index 26b6185520..5eaf25acc5 100644 --- a/src/SHAMapSync.h +++ b/src/SHAMapSync.h @@ -15,12 +15,12 @@ public: const std::vector& nodeData, bool isLeaf) { // WRITEME: If 'isLeaf' is true, this is a transaction - theApp->getNodeCache().store(nodeHash, nodeData); + theApp->getTempNodeCache().store(nodeHash, nodeData); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { // WRITEME: We could check our own map, we could check transaction tables - return theApp->getNodeCache().retrieve(nodeHash, nodeData); + return theApp->getTempNodeCache().retrieve(nodeHash, nodeData); } }; @@ -40,11 +40,8 @@ public: theApp->getHashedObjectStore().store(ACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); - if (!node) return false; - nodeData = node->getData(); - return true; + { // fetchNode already tried + return false; } }; @@ -64,11 +61,8 @@ public: theApp->getHashedObjectStore().store(isLeaf ? TRANSACTION : TRANSACTION_NODE, mLedgerSeq, nodeData, nodeHash); } virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(nodeHash); - if (!node) return false; - nodeData = node->getData(); - return true; + { // fetchNode already tried + return false; } }; From 23fd638b2565def52e943dd1c80521050082a399 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 14:06:47 -0700 Subject: [PATCH 31/36] Bugfix. Sorry. --- src/LedgerHistory.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index e42e13020e..2d260c0ee0 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -60,15 +60,18 @@ Ledger::pointer LedgerHistory::getLedgerByHash(const uint256& hash) Ledger::pointer ret = mLedgersByHash.fetch(hash); if (ret) return ret; +#if 0 // FIXME: A ledger without SHA maps isn't very useful // This code will need to build them -// ret = Ledger::loadByHash(hash); -// if (!ret) return ret; -// assert(ret->getHash() == hash); + ret = Ledger::loadByHash(hash); + if (!ret) return ret; + assert(ret->getHash() == hash); boost::recursive_mutex::scoped_lock sl(mLedgersByHash.peekMutex()); mLedgersByHash.canonicalize(hash, ret); if (ret->isAccepted()) mLedgersByIndex[ret->getLedgerSeq()] = ret; +#endif + return ret; } From ef87efb96d0fb1ba46f7d85c4e2f2d69b4fdf942 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 14:34:19 -0700 Subject: [PATCH 32/36] Explain how we're going to fix getLedgerByHash --- src/LedgerHistory.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index 2d260c0ee0..8f9b1bec9e 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -63,6 +63,11 @@ Ledger::pointer LedgerHistory::getLedgerByHash(const uint256& hash) #if 0 // FIXME: A ledger without SHA maps isn't very useful // This code will need to build them + + // The fix is to audit all callers to this function and replace them with + // higher-level functions that ask more-specific questions that we can + // test against our database + ret = Ledger::loadByHash(hash); if (!ret) return ret; assert(ret->getHash() == hash); From 6018cba00ecb2a104180f43f6fc3b3d12e1571bd Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 16:29:52 -0700 Subject: [PATCH 33/36] Track server and protocol version. --- src/Version.h | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/Version.h diff --git a/src/Version.h b/src/Version.h new file mode 100644 index 0000000000..1afd390afd --- /dev/null +++ b/src/Version.h @@ -0,0 +1,21 @@ + +// Versions + +#ifndef SERVER_VERSION_MAJ + +#define SERVER_VERSION_MAJ 0 +#define SERVER_VERSION_MIN 1 +#define SERVER_VERSION_SUB "-a" +#define SERVER_NAME "NewCoin" + +#define SV_STRINGIZE(x) #x +#define SERVER_VERSION \ + (SERVER_NAME "-" SV_STRINGIZE(SERVER_VERSION_MAJ) "." SV_STRINGIZE(SERVER_VERSION_MIN) SERVER_VERSION_SUB) + +#define PROTO_VERSION_MAJ 0 +#define PROTO_VERSION_MIN 0 + +#define MIN_PROTO_MAJ 0 +#define MIN_PROTO_MIN 0 + +#endif From 69d7b8a93d77435e0dc79f53f33339f39dbfee40 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 16:30:13 -0700 Subject: [PATCH 34/36] Track server and protocol versions and minimum supported versions. This is a protocol-breaking change. It's needed because other protocol-breaking changes are needed for security reasons, and we don't want subtly-incompatible nodes connecting. --- src/Config.cpp | 2 -- src/Config.h | 4 ---- src/Peer.cpp | 34 ++++++++++++++++++++++++++-------- src/newcoin.proto | 22 ++++++++++++++-------- src/rpc.cpp | 7 ++----- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 41030c187b..6e4b67ce87 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -112,8 +112,6 @@ void Config::setup(const std::string& strConf) // Defaults // - VERSION = 1; - NETWORK_START_TIME = 1319844908; PEER_PORT = SYSTEM_PEER_PORT; diff --git a/src/Config.h b/src/Config.h index f7a443b96d..6a88d13cfe 100644 --- a/src/Config.h +++ b/src/Config.h @@ -43,10 +43,6 @@ const int SYSTEM_WEBSOCKET_PORT = 6562; class Config { public: - // Core software parameters - int VERSION; - std::string VERSION_STR; - // Configuration parameters boost::filesystem::path CONFIG_FILE; boost::filesystem::path CONFIG_DIR; diff --git a/src/Peer.cpp b/src/Peer.cpp index f4cbbcb0a9..eb9ad2099e 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -8,6 +8,7 @@ #include "../json/writer.h" +#include "Version.h" #include "Peer.h" #include "Config.h" #include "Application.h" @@ -565,15 +566,20 @@ void Peer::processReadBuffer() void Peer::recvHello(newcoin::TMHello& packet) { -#ifdef DEBUG - Log(lsINFO) << "Recv(Hello) v=" << packet.version() << ", index=" << packet.ledgerindex(); -#endif + Log(lsTRACE) << "Recv(Hello) v=" << packet.versionmajor() << "." << packet.versionminor(); bool bDetach = true; // Cancel verification timeout. (void) mVerifyTimer.cancel(); - if (!mNodePublic.setNodePublic(packet.nodepublic())) + if ((packet.minprotoversionmajor() > PROTO_VERSION_MAJ) || + ((packet.minprotoversionmajor() == PROTO_VERSION_MAJ) && (packet.minprotoversionminor() > PROTO_VERSION_MIN))) + { + Log(lsINFO) << "Recv(Hello): Server requires protocol version " << + packet.minprotoversionmajor() << "." << packet.minprotoversionminor() << " we run " << + PROTO_VERSION_MAJ << "." << PROTO_VERSION_MIN; + } + else if (!mNodePublic.setNodePublic(packet.nodepublic())) { Log(lsINFO) << "Recv(Hello): Disconnect: Bad node public key."; } @@ -585,6 +591,14 @@ void Peer::recvHello(newcoin::TMHello& packet) { // Successful connection. Log(lsINFO) << "Recv(Hello): Connect: " << mNodePublic.humanNodePublic(); + if ( (packet.versionmajor() != SERVER_VERSION_MAJ) || (packet.versionminor() != SERVER_VERSION_MIN)) + { + if (packet.has_fullversion()) + Log(lsINFO) << " Peer is running version " << packet.fullversion(); + else + Log(lsINFO) << " Peer is running version " << packet.versionmajor() << "." << packet.versionminor(); + } + if (mClientConnect) { // If we connected due to scan, no longer need to scan. @@ -1104,16 +1118,20 @@ void Peer::sendHello() newcoin::TMHello h; - h.set_version(theConfig.VERSION); - h.set_ledgerindex(theApp->getOPs().getCurrentLedgerID()); + h.set_versionmajor(SERVER_VERSION_MAJ); + h.set_versionminor(SERVER_VERSION_MIN); + h.set_protoversionmajor(PROTO_VERSION_MAJ); + h.set_protoversionminor(PROTO_VERSION_MIN); + h.set_minprotoversionminor(MIN_PROTO_MAJ); + h.set_minprotoversionmajor(MIN_PROTO_MIN); + h.set_fullversion(SERVER_VERSION); h.set_nettime(theApp->getOPs().getNetworkTimeNC()); h.set_nodepublic(theApp->getWallet().getNodePublic().humanNodePublic()); h.set_nodeproof(&vchSig[0], vchSig.size()); h.set_ipv4port(theConfig.PEER_PORT); Ledger::pointer closedLedger = theApp->getMasterLedger().getClosedLedger(); - assert(closedLedger && closedLedger->isClosed()); - if (closedLedger->isClosed()) + if (closedLedger && closedLedger->isClosed()) { uint256 hash = closedLedger->getHash(); h.set_closedledger(hash.begin(), hash.GetSerializeSize()); diff --git a/src/newcoin.proto b/src/newcoin.proto index 1b4e30f556..99d9b591fb 100644 --- a/src/newcoin.proto +++ b/src/newcoin.proto @@ -37,14 +37,20 @@ enum MessageType { message TMHello { - required uint32 version = 1; - optional uint32 ledgerIndex = 2; - optional uint64 netTime = 3; - optional bytes nodePublic = 4; // node may opt to remain anonymous - optional bytes nodeProof = 5; - optional uint32 ipv4Port = 6; - optional bytes closedLedger = 7; // our last closed ledger - optional bytes previousLedger = 8; // the ledger before the last closed ledger + required uint32 versionMinor = 1; + required uint32 versionMajor = 2; + required uint32 protoVersionMinor = 3; + required uint32 protoVersionMajor = 4; + required uint32 minProtoVersionMinor = 5; + required uint32 minProtoVersionMajor = 6; + required bytes nodePublic = 7; + required bytes nodeProof = 8; + optional string fullVersion = 9; + optional uint64 netTime = 10; + optional uint32 ipv4Port = 11; + optional uint32 ledgerIndex = 12; + optional bytes closedLedger = 13; // our last closed ledger + optional bytes previousLedger = 14; // the ledger before the last closed ledger } diff --git a/src/rpc.cpp b/src/rpc.cpp index 77a414cc4b..2a289197de 100644 --- a/src/rpc.cpp +++ b/src/rpc.cpp @@ -15,14 +15,11 @@ #include "BitcoinUtil.h" #include "Config.h" #include "Log.h" +#include "Version.h" using namespace boost; using namespace boost::asio; - - - - Json::Value JSONRPCError(int code, const std::string& message) { Json::Value error(Json::objectValue); @@ -116,7 +113,7 @@ std::string HTTPReply(int nStatus, const std::string& strMsg) rfc1123Time().c_str(), access.c_str(), strMsg.size(), - theConfig.VERSION_STR.c_str(), + SERVER_VERSION, strMsg.c_str()); } From 808cec521e1ced3582d645fd8afb6c7ad8002af5 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 16:46:45 -0700 Subject: [PATCH 35/36] It's crap like that that makes me dislike C++ sometimes. --- src/Version.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Version.h b/src/Version.h index 1afd390afd..e9fe8bc151 100644 --- a/src/Version.h +++ b/src/Version.h @@ -8,7 +8,8 @@ #define SERVER_VERSION_SUB "-a" #define SERVER_NAME "NewCoin" -#define SV_STRINGIZE(x) #x +#define SV_STRINGIZE(x) SV_STRINGIZE2(x) +#define SV_STRINGIZE2(x) #x #define SERVER_VERSION \ (SERVER_NAME "-" SV_STRINGIZE(SERVER_VERSION_MAJ) "." SV_STRINGIZE(SERVER_VERSION_MIN) SERVER_VERSION_SUB) From b4eea47a106a931ada40df3fbde3115f24e4fc40 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 24 Jun 2012 16:47:47 -0700 Subject: [PATCH 36/36] Track peer versions and put in the Json output. --- src/Peer.cpp | 26 +++++++++++++++++++------- src/Peer.h | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Peer.cpp b/src/Peer.cpp index eb9ad2099e..ea677631ca 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -590,6 +590,7 @@ void Peer::recvHello(newcoin::TMHello& packet) else { // Successful connection. Log(lsINFO) << "Recv(Hello): Connect: " << mNodePublic.humanNodePublic(); + mHello = packet; if ( (packet.versionmajor() != SERVER_VERSION_MAJ) || (packet.versionminor() != SERVER_VERSION_MIN)) { @@ -1159,13 +1160,24 @@ void Peer::punishPeer(PeerPunish) { } -Json::Value Peer::getJson() { - Json::Value ret(Json::objectValue); +Json::Value Peer::getJson() +{ + Json::Value ret(Json::objectValue); - ret["this"] = ADDRESS(this); - ret["public_key"] = mNodePublic.ToString(); - ret["ip"] = mIpPortConnect.first; - ret["port"] = mIpPortConnect.second; + ret["this"] = ADDRESS(this); + ret["public_key"] = mNodePublic.ToString(); + ret["ip"] = mIpPortConnect.first; + ret["port"] = mIpPortConnect.second; + + if (mHello.has_fullversion()) + ret["version"] = mHello.fullversion(); + else if (mHello.has_versionminor() && mHello.has_versionmajor()) + ret["version"] = boost::lexical_cast(mHello.versionmajor()) + "." + + boost::lexical_cast(mHello.versionminor()); + + if (mHello.has_protoversionminor() && mHello.has_protoversionmajor()) + ret["protocol"] = boost::lexical_cast(mHello.protoversionmajor()) + "." + + boost::lexical_cast(mHello.protoversionminor()); if (!mIpPort.first.empty()) { @@ -1173,7 +1185,7 @@ Json::Value Peer::getJson() { ret["verified_port"] = mIpPort.second; } - return ret; + return ret; } // vim:ts=4 diff --git a/src/Peer.h b/src/Peer.h index 3150532efa..3223632904 100644 --- a/src/Peer.h +++ b/src/Peer.h @@ -55,6 +55,7 @@ protected: std::list mSendQ; PackedMessage::pointer mSendingPacket; newcoin::TMStatusChange mLastStatus; + newcoin::TMHello mHello; Peer(boost::asio::io_service& io_service, boost::asio::ssl::context& ctx);