From 4faad381120f2bae2cee8dccc7e6347598589e95 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Sep 2012 16:29:42 -0700 Subject: [PATCH 01/44] Base classes used polymorphically whose child classes have data members or destructors must themselves have virtual destructors or Bad Things(TM) happen. --- src/Operation.h | 2 ++ src/ScriptData.h | 2 ++ src/TransactionMeta.h | 1 + 3 files changed, 5 insertions(+) diff --git a/src/Operation.h b/src/Operation.h index 9fca2498c4..11c37d38f3 100644 --- a/src/Operation.h +++ b/src/Operation.h @@ -12,6 +12,8 @@ public: virtual bool work(Interpreter* interpreter)=0; virtual int getFee(); + + virtual ~Operation() { ; } }; // this is just an Int in the code diff --git a/src/ScriptData.h b/src/ScriptData.h index af9ce45c0e..66fe2ed3e4 100644 --- a/src/ScriptData.h +++ b/src/ScriptData.h @@ -9,6 +9,8 @@ class Data public: typedef boost::shared_ptr pointer; + virtual ~Data(){ ; } + virtual bool isInt32(){ return(false); } virtual bool isFloat(){ return(false); } virtual bool isUint160(){ return(false); } diff --git a/src/TransactionMeta.h b/src/TransactionMeta.h index 729a724808..b62f9f3617 100644 --- a/src/TransactionMeta.h +++ b/src/TransactionMeta.h @@ -46,6 +46,7 @@ protected: public: TransactionMetaNodeEntry(int type) : mType(type) { ; } + virtual ~TransactionMetaNodeEntry() { ; } int getType() const { return mType; } virtual Json::Value getJson(int) const = 0; From 2c2d9cf8f50348e543d19a36f8266722f8fd50c9 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Sep 2012 16:30:25 -0700 Subject: [PATCH 02/44] Small cleanups. --- src/SerializedTransaction.h | 2 +- src/bignum.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SerializedTransaction.h b/src/SerializedTransaction.h index 59f1650ca1..ee4ffb3ea4 100644 --- a/src/SerializedTransaction.h +++ b/src/SerializedTransaction.h @@ -28,7 +28,7 @@ protected: TransactionType mType; STVariableLength mSignature; STObject mMiddleTxn, mInnerTxn; - TransactionFormat* mFormat; + const TransactionFormat* mFormat; SerializedTransaction* duplicate() const { return new SerializedTransaction(*this); } diff --git a/src/bignum.h b/src/bignum.h index 78bce87609..a9a08eda2d 100644 --- a/src/bignum.h +++ b/src/bignum.h @@ -27,7 +27,7 @@ private: protected: BN_CTX* pctx; - BN_CTX* operator=(BN_CTX* pnew) { return pctx = pnew; } + CAutoBN_CTX& operator=(BN_CTX* pnew) { pctx = pnew; return *this; } public: CAutoBN_CTX() From 81cd4cf82017ff2507efcb720d1bb8cbd6ac7fbc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 11 Sep 2012 11:56:41 -0700 Subject: [PATCH 03/44] Start of code to track network state, track overwhelmed nodes, and adjust transaction fees. --- src/LedgerConsensus.cpp | 2 +- src/LedgerTiming.h | 4 ++++ src/ValidationCollection.cpp | 27 ++++++++++++++++++++++++--- src/ValidationCollection.h | 4 +++- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index ce064fe47c..73bc3e6821 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -688,7 +688,7 @@ bool LedgerConsensus::haveConsensus() else ++disagree; } - int currentValidations = theApp->getValidations().getCurrentValidationCount(mPreviousLedger->getCloseTimeNC()); + int currentValidations = theApp->getValidations().getNodesAfter(mPrevLedgerHash); return ContinuousLedgerTiming::haveConsensus(mPreviousProposers, agree + disagree, agree, currentValidations, mPreviousMSeconds, mCurrentMSeconds); } diff --git a/src/LedgerTiming.h b/src/LedgerTiming.h index 80e89b30f4..551c9bfce6 100644 --- a/src/LedgerTiming.h +++ b/src/LedgerTiming.h @@ -28,6 +28,10 @@ // How often we check state or change positions (in milliseconds) # define LEDGER_GRANULARITY 1000 +// The percentage of active trusted validators that must be able to +// keep up with the network or we consider the network overloaded +# define LEDGER_NET_RATIO 70 + // How long we consider a proposal fresh # define PROPOSE_FRESHNESS 20 diff --git a/src/ValidationCollection.cpp b/src/ValidationCollection.cpp index b11b38b633..e4d1117549 100644 --- a/src/ValidationCollection.cpp +++ b/src/ValidationCollection.cpp @@ -116,19 +116,40 @@ int ValidationCollection::getTrustedValidationCount(const uint256& ledger) return trusted; } -int ValidationCollection::getCurrentValidationCount(uint32 afterTime) -{ +int ValidationCollection::getNodesAfter(const uint256& ledger) +{ // Number of trusted nodes that have moved past this ledger int count = 0; boost::mutex::scoped_lock sl(mValidationLock); for (boost::unordered_map::iterator it = mCurrentValidations.begin(), end = mCurrentValidations.end(); it != end; ++it) { - if (it->second->isTrusted() && (it->second->getSignTime() > afterTime)) + if (it->second->isTrusted() && it->second->isPreviousHash(ledger)) ++count; } return count; } +int ValidationCollection::getLoadRatio(bool overLoaded) +{ // how many trusted nodes are able to keep up, higher is better + int goodNodes = overLoaded ? 1 : 0; + int badNodes = overLoaded ? 0 : 1; + { + boost::mutex::scoped_lock sl(mValidationLock); + for (boost::unordered_map::iterator it = mCurrentValidations.begin(), + end = mCurrentValidations.end(); it != end; ++it) + { + if (it->second->isTrusted()) + { + if (it->second->isFull()) + ++goodNodes; + else + ++badNodes; + } + } + } + return (goodNodes * 100) / (goodNodes + badNodes); +} + boost::unordered_map ValidationCollection::getCurrentValidations(uint256 currentLedger) { uint32 cutoff = theApp->getOPs().getNetworkTimeNC() - LEDGER_VAL_INTERVAL; diff --git a/src/ValidationCollection.h b/src/ValidationCollection.h index 1972b7b63e..95cd4592e0 100644 --- a/src/ValidationCollection.h +++ b/src/ValidationCollection.h @@ -34,7 +34,9 @@ public: void getValidationCount(const uint256& ledger, bool currentOnly, int& trusted, int& untrusted); int getTrustedValidationCount(const uint256& ledger); - int getCurrentValidationCount(uint32 afterTime); + + int getNodesAfter(const uint256& ledger); + int getLoadRatio(bool overLoaded); boost::unordered_map getCurrentValidations(uint256 currentLedger = uint256()); From ca1436ac2593957c8ecbb102678f1e6841b238d5 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 11 Sep 2012 14:51:13 -0700 Subject: [PATCH 04/44] Remove some dead code that was in my way. --- src/LedgerMaster.cpp | 3 +-- src/LedgerMaster.h | 3 +-- src/NetworkOPs.cpp | 6 ++---- src/NetworkOPs.h | 3 +-- src/Peer.cpp | 8 +------- src/SerializedObject.h | 1 - src/Transaction.h | 1 - src/newcoin.proto | 3 --- 8 files changed, 6 insertions(+), 22 deletions(-) diff --git a/src/LedgerMaster.cpp b/src/LedgerMaster.cpp index a3cc5a1475..abdbf12e99 100644 --- a/src/LedgerMaster.cpp +++ b/src/LedgerMaster.cpp @@ -80,8 +80,7 @@ Ledger::pointer LedgerMaster::closeLedger() return closingLedger; } -TER LedgerMaster::doTransaction(const SerializedTransaction& txn, uint32 targetLedger, - TransactionEngineParams params) +TER LedgerMaster::doTransaction(const SerializedTransaction& txn, TransactionEngineParams params) { TER result = mEngine.applyTransaction(txn, params); theApp->getOPs().pubTransaction(mEngine.getLedger(), txn, result); diff --git a/src/LedgerMaster.h b/src/LedgerMaster.h index 8db6f34490..37db69c5c2 100644 --- a/src/LedgerMaster.h +++ b/src/LedgerMaster.h @@ -45,8 +45,7 @@ public: void runStandAlone() { mFinalizedLedger = mCurrentLedger; } - TER doTransaction(const SerializedTransaction& txn, uint32 targetLedger, - TransactionEngineParams params); + TER doTransaction(const SerializedTransaction& txn, TransactionEngineParams params); void pushLedger(Ledger::ref newLedger); void pushLedger(Ledger::ref newLCL, Ledger::ref newOL); diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 754559db83..cf3873ba89 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -93,7 +93,7 @@ Transaction::pointer NetworkOPs::submitTransaction(const Transaction::pointer& t return tpTransNew; } -Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, uint32 tgtLedger, Peer* source) +Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, Peer* source) { Transaction::pointer dbtx = theApp->getMasterTransaction().fetch(trans->getID(), true); if (dbtx) return dbtx; @@ -105,7 +105,7 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, return trans; } - TER r = mLedgerMaster->doTransaction(*trans->getSTransaction(), tgtLedger, tapOPEN_LEDGER); + TER r = mLedgerMaster->doTransaction(*trans->getSTransaction(), tapOPEN_LEDGER); if (r == tefFAILURE) throw Fault(IO_ERROR); if (r == terPRE_SEQ) @@ -140,7 +140,6 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, tx.set_rawtransaction(&s.getData().front(), s.getLength()); tx.set_status(newcoin::tsCURRENT); tx.set_receivetimestamp(getNetworkTimeNC()); - tx.set_ledgerindexpossible(trans->getLedger()); PackedMessage::pointer packet = boost::make_shared(tx, newcoin::mtTRANSACTION); theApp->getConnectionPool().relayMessage(source, packet); @@ -157,7 +156,6 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, tx.set_rawtransaction(&s.getData().front(), s.getLength()); tx.set_status(newcoin::tsCURRENT); tx.set_receivetimestamp(getNetworkTimeNC()); - tx.set_ledgerindexpossible(tgtLedger); PackedMessage::pointer packet = boost::make_shared(tx, newcoin::mtTRANSACTION); theApp->getConnectionPool().relayMessage(source, packet); } diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 401d33baee..04f4e731ae 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -116,8 +116,7 @@ public: // Transaction::pointer submitTransaction(const Transaction::pointer& tpTrans); - Transaction::pointer processTransaction(Transaction::pointer transaction, uint32 targetLedger = 0, - Peer* source = NULL); + Transaction::pointer processTransaction(Transaction::pointer transaction, Peer* source = NULL); Transaction::pointer findTransactionByID(const uint256& transactionID); int findTransactionsBySource(const uint256& uLedger, std::list&, const NewcoinAddress& sourceAccount, uint32 minSeq, uint32 maxSeq); diff --git a/src/Peer.cpp b/src/Peer.cpp index aec3ce31e2..ea27e06c81 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -702,13 +702,7 @@ void Peer::recvTransaction(newcoin::TMTransaction& packet) } #endif - uint32 targetLedger = 0; - if (packet.has_ledgerindexfinal()) - targetLedger = packet.ledgerindexfinal(); - else if (packet.has_ledgerindexpossible()) - targetLedger = packet.ledgerindexpossible(); - - tx = theApp->getOPs().processTransaction(tx, targetLedger, this); + tx = theApp->getOPs().processTransaction(tx, this); if(tx->getStatus() != INCLUDED) { // transaction wasn't accepted into ledger diff --git a/src/SerializedObject.h b/src/SerializedObject.h index c8245d6d9d..08377304c1 100644 --- a/src/SerializedObject.h +++ b/src/SerializedObject.h @@ -106,7 +106,6 @@ enum SOE_Field sfTakerGets, sfTakerPays, sfTarget, - sfTargetLedger, sfTransferRate, sfVersion, sfWalletLocator, diff --git a/src/Transaction.h b/src/Transaction.h index 1ae8c0aa43..abe612853f 100644 --- a/src/Transaction.h +++ b/src/Transaction.h @@ -281,7 +281,6 @@ public: STAmount getAmount() const { return mTransaction->getITFieldU64(sfAmount); } STAmount getFee() const { return mTransaction->getTransactionFee(); } uint32 getFromAccountSeq() const { return mTransaction->getSequence(); } - uint32 getSourceLedger() const { return mTransaction->getITFieldU32(sfTargetLedger); } uint32 getIdent() const { return mTransaction->getITFieldU32(sfSourceTag); } std::vector getSignature() const { return mTransaction->getSignature(); } uint32 getLedger() const { return mInLedger; } diff --git a/src/newcoin.proto b/src/newcoin.proto index 2426f16ece..cd33de9f0f 100644 --- a/src/newcoin.proto +++ b/src/newcoin.proto @@ -69,9 +69,6 @@ message TMTransaction { required bytes rawTransaction = 1; required TransactionStatus status = 2; optional uint64 receiveTimestamp = 3; - optional uint32 ledgerIndexPossible = 4; // the node may not know - optional uint32 ledgerIndexFinal = 5; - optional bytes conflictingTransaction = 6; } From b5da6c22a52dc875c801b30300175be992984652 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 11 Sep 2012 15:35:23 -0700 Subject: [PATCH 05/44] Missing piece for TX metadata. --- src/SHAMapNodes.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 65514ad95f..746176aba3 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -296,8 +296,9 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector(txID, s.peekData()); mType = tnTRANSACTION_MD; } From ca6e9cf764a8247845368a89a079f025dc6cab9c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 11 Sep 2012 21:36:23 -0700 Subject: [PATCH 06/44] Fix a case where we can get stuck in the wrong consensus window and have to wait for it to timeout. --- src/LedgerConsensus.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 73bc3e6821..515722b408 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -333,6 +333,7 @@ void LedgerConsensus::handleLCL(const uint256& lclHash) mPeerPositions.clear(); mDisputes.clear(); mDeadNodes.clear(); + playbackProposals(); return; } From 30cd0e197d9611f4bcaa21c8cbf94ff926d7ae7c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 12 Sep 2012 09:18:03 -0700 Subject: [PATCH 07/44] Remove dead code that's in my way. --- src/Ledger.cpp | 12 ------------ src/Ledger.h | 1 - 2 files changed, 13 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 71a82231dc..1885439f29 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -211,18 +211,6 @@ RippleState::pointer Ledger::accessRippleState(const uint256& uNode) return boost::make_shared(sle); } -bool Ledger::addTransaction(const Transaction::pointer& trans) -{ // low-level - just add to table - assert(!mAccepted); - assert(trans->getID().isNonZero()); - Serializer s; - trans->getSTransaction()->add(s); - SHAMapItem::pointer item = boost::make_shared(trans->getID(), s.peekData()); - if (!mTransactionMap->addGiveItem(item, true, false)) // FIXME: TX metadata - return false; - return true; -} - bool Ledger::addTransaction(const uint256& txID, const Serializer& txn) { // low-level - just add to table SHAMapItem::pointer item = boost::make_shared(txID, txn.peekData()); diff --git a/src/Ledger.h b/src/Ledger.h index c671a57bf2..a0a274e94e 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -82,7 +82,6 @@ private: protected: - bool addTransaction(const Transaction::pointer&); bool addTransaction(const uint256& id, const Serializer& txn); static Ledger::pointer getSQL(const std::string& sqlStatement); From 1ba5b02f141ce043d09e7385b8e78bee88fab5ab Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 12 Sep 2012 09:18:10 -0700 Subject: [PATCH 08/44] Don't track metadata for directory nodes. --- src/LedgerEntrySet.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 3c5adc432a..6196a3a802 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -350,8 +350,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) nType = TMNCreatedNode; break; - default: - // ignore these + default: // ignore these break; } @@ -359,6 +358,9 @@ void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) continue; SLE::pointer origNode = origLedger->getSLE(it->first); + if (origNode->getType() == ltDIR_NODE) // No metadata for dir nodes + continue; + SLE::pointer curNode = it->second.mEntry; TransactionMetaNode &metaNode = mSet.getAffectedNode(it->first, nType); From d2336e3eea2d9d27e417d9aae1efef6c52d944ff Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 12 Sep 2012 18:40:16 -0700 Subject: [PATCH 09/44] Ledger functions to handle transaction metadata. --- src/Ledger.cpp | 49 +++++++++++++++++++++++++++++++++++++++++-- src/Ledger.h | 7 +++++-- src/SHAMap.cpp | 10 +++++++++ src/SHAMap.h | 1 + src/TransactionMeta.h | 3 +++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 1885439f29..00b647c8b6 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -214,7 +214,18 @@ RippleState::pointer Ledger::accessRippleState(const uint256& uNode) bool Ledger::addTransaction(const uint256& txID, const Serializer& txn) { // low-level - just add to table SHAMapItem::pointer item = boost::make_shared(txID, txn.peekData()); - if (!mTransactionMap->addGiveItem(item, true, false)) // FIXME: TX metadata + if (!mTransactionMap->addGiveItem(item, true, false)) + return false; + return true; +} + +bool Ledger::addTransaction(const uint256& txID, const Serializer& txn, const Serializer& md) +{ // low-level - just add to table + Serializer s(txn.getDataLength() + md.getDataLength() + 64); + s.addVL(txn.peekData()); + s.addVL(md.peekData()); + SHAMapItem::pointer item = boost::make_shared(txID, s.peekData()); + if (!mTransactionMap->addGiveItem(item, true, true)) return false; return true; } @@ -225,7 +236,8 @@ Transaction::pointer Ledger::getTransaction(const uint256& transID) const if (!item) return Transaction::pointer(); Transaction::pointer txn = theApp->getMasterTransaction().fetch(transID, false); - if (txn) return txn; + if (txn) + return txn; txn = Transaction::sharedTransaction(item->getData(), true); if (txn->getStatus() == NEW) @@ -235,6 +247,39 @@ Transaction::pointer Ledger::getTransaction(const uint256& transID) const return txn; } +bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, TransactionMetaSet::pointer& meta) +{ + SHAMapTreeNode::TNType type; + SHAMapItem::pointer item = mTransactionMap->peekItem(txID, type); + if (!item) + return false; + + if (type == SHAMapTreeNode::tnTRANSACTION_NM) + { // in tree with no metadata + txn = theApp->getMasterTransaction().fetch(txID, false); + meta = TransactionMetaSet::pointer(); + if (!txn) + txn = Transaction::sharedTransaction(item->getData(), true); + } + else if (type == SHAMapTreeNode::tnTRANSACTION_MD) + { // in tree with metadata + SerializerIterator it(item->getData()); + txn = theApp->getMasterTransaction().fetch(txID, false); + if (!txn) + txn = Transaction::sharedTransaction(it.getVL(), true); + else + it.getVL(); // skip transaction + meta = boost::make_shared(mLedgerSeq, it.getVL()); + } + else + return false; + + if (txn->getStatus() == NEW) + txn->setStatus(mClosed ? COMMITTED : INCLUDED, mLedgerSeq); + theApp->getMasterTransaction().canonicalize(txn, false); + return true; +} + bool Ledger::unitTest() { return true; diff --git a/src/Ledger.h b/src/Ledger.h index a0a274e94e..d059f01581 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -11,6 +11,7 @@ #include "../json/value.h" #include "Transaction.h" +#include "TransactionMeta.h" #include "AccountState.h" #include "RippleState.h" #include "NicknameState.h" @@ -56,7 +57,7 @@ public: TR_PASTASEQ = 6, // account is past this transaction TR_PREASEQ = 7, // account is missing transactions before this TR_BADLSEQ = 8, // ledger too early - TR_TOOSMALL = 9, // amount is less than Tx fee + TR_TOOSMALL = 9, // amount is less than Tx fee }; // ledger close flags @@ -82,7 +83,6 @@ private: protected: - bool addTransaction(const uint256& id, const Serializer& txn); static Ledger::pointer getSQL(const std::string& sqlStatement); @@ -151,8 +151,11 @@ public: bool isAcquiringAS(void); // Transaction Functions + bool addTransaction(const uint256& id, const Serializer& txn); + bool addTransaction(const uint256& id, const Serializer& txn, const Serializer& metaData); bool hasTransaction(const uint256& TransID) const { return mTransactionMap->hasItem(TransID); } Transaction::pointer getTransaction(const uint256& transID) const; + bool getTransaction(const uint256& transID, Transaction::pointer& txn, TransactionMetaSet::pointer& txMeta); // high-level functions AccountState::pointer getAccountState(const NewcoinAddress& acctID); diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 8ae69c405e..8a232ec547 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -421,6 +421,16 @@ SHAMapItem::pointer SHAMap::peekItem(const uint256& id) return leaf->peekItem(); } +SHAMapItem::pointer SHAMap::peekItem(const uint256& id, SHAMapTreeNode::TNType& type) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + SHAMapTreeNode* leaf = walkToPointer(id); + if (!leaf) + return SHAMapItem::pointer(); + type = leaf->getType(); + return leaf->peekItem(); +} + bool SHAMap::hasItem(const uint256& id) { // does the tree have an item with this ID boost::recursive_mutex::scoped_lock sl(mLock); diff --git a/src/SHAMap.h b/src/SHAMap.h index b84dcb7272..0f7061d71a 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -318,6 +318,7 @@ public: // save a copy if you only need a temporary SHAMapItem::pointer peekItem(const uint256& id); + SHAMapItem::pointer peekItem(const uint256& id, SHAMapTreeNode::TNType& type); // traverse functions SHAMapItem::pointer peekFirstItem(); diff --git a/src/TransactionMeta.h b/src/TransactionMeta.h index b62f9f3617..10423aeda9 100644 --- a/src/TransactionMeta.h +++ b/src/TransactionMeta.h @@ -168,6 +168,9 @@ public: class TransactionMetaSet { +public: + typedef boost::shared_ptr pointer; + protected: uint256 mTransactionID; uint32 mLedger; From 953f0ad63faf2d2f7ee058a5638dfadfd0a6f7a0 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:01:40 -0700 Subject: [PATCH 10/44] Don't charge for transactions twice. Have the "old" getTransaction handle metadata sanely. Fix calcRawMeta to fit the new model where an LES holds a reference to its ledger Don't put metadata in open ledger txn sets to avoid breaking the proposal mechanism. --- src/Ledger.cpp | 20 ++++++++++++++++++-- src/LedgerEntrySet.cpp | 10 +++++----- src/LedgerEntrySet.h | 2 +- src/TransactionEngine.cpp | 23 ++++++++++++++++------- 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 00b647c8b6..f106297d4d 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -232,14 +232,30 @@ bool Ledger::addTransaction(const uint256& txID, const Serializer& txn, const Se Transaction::pointer Ledger::getTransaction(const uint256& transID) const { - SHAMapItem::pointer item = mTransactionMap->peekItem(transID); + SHAMapTreeNode::TNType type; + SHAMapItem::pointer item = mTransactionMap->peekItem(transID, type); if (!item) return Transaction::pointer(); Transaction::pointer txn = theApp->getMasterTransaction().fetch(transID, false); if (txn) return txn; - txn = Transaction::sharedTransaction(item->getData(), true); + if (type == SHAMapTreeNode::tnTRANSACTION_NM) + txn = Transaction::sharedTransaction(item->getData(), true); + else if (type == SHAMapTreeNode::tnTRANSACTION_MD) + { + std::vector txnData; + int txnLength; + if (!item->peekSerializer().getVL(txnData, 0, txnLength)) + return Transaction::pointer(); + txn = Transaction::sharedTransaction(txnData, false); + } + else + { + assert(false); + return Transaction::pointer(); + } + if (txn->getStatus() == NEW) txn->setStatus(mClosed ? COMMITTED : INCLUDED, mLedgerSeq); diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 6196a3a802..5c3f16f732 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -325,7 +325,7 @@ bool LedgerEntrySet::threadOwners(TransactionMetaNode& metaNode, SLE::ref node, return false; } -void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) +void LedgerEntrySet::calcRawMeta(Serializer& s) { // calculate the raw meta data and return it. This must be called before the set is committed // Entries modified only as a result of building the transaction metadata @@ -357,7 +357,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) if (nType == TMNEndOfMetadata) continue; - SLE::pointer origNode = origLedger->getSLE(it->first); + SLE::pointer origNode = mLedger->getSLE(it->first); if (origNode->getType() == ltDIR_NODE) // No metadata for dir nodes continue; @@ -366,7 +366,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) if (nType == TMNDeletedNode) { - threadOwners(metaNode, origNode, origLedger, newMod); + threadOwners(metaNode, origNode, mLedger, newMod); if (origNode->getIFieldPresent(sfAmount)) { // node has an amount, covers ripple state nodes @@ -398,12 +398,12 @@ void LedgerEntrySet::calcRawMeta(Serializer& s, Ledger::ref origLedger) } if (nType == TMNCreatedNode) // if created, thread to owner(s) - threadOwners(metaNode, curNode, origLedger, newMod); + threadOwners(metaNode, curNode, mLedger, newMod); if ((nType == TMNCreatedNode) || (nType == TMNModifiedNode)) { if (curNode->isThreadedType()) // always thread to self - threadTx(metaNode, curNode, origLedger, newMod); + threadTx(metaNode, curNode, mLedger, newMod); } if (nType == TMNModifiedNode) diff --git a/src/LedgerEntrySet.h b/src/LedgerEntrySet.h index 6abef7cab3..92d96294e9 100644 --- a/src/LedgerEntrySet.h +++ b/src/LedgerEntrySet.h @@ -120,7 +120,7 @@ public: STAmount accountFunds(const uint160& uAccountID, const STAmount& saDefault); Json::Value getJson(int) const; - void calcRawMeta(Serializer&, Ledger::ref originalLedger); + void calcRawMeta(Serializer&); // iterator functions bool isEmpty() const { return mEntries.empty(); } diff --git a/src/TransactionEngine.cpp b/src/TransactionEngine.cpp index 2e5e22bd88..1cf9379753 100644 --- a/src/TransactionEngine.cpp +++ b/src/TransactionEngine.cpp @@ -430,10 +430,10 @@ TER TransactionEngine::applyTransaction(const SerializedTransaction& txn, Transa break; case ttCONTRACT: - terResult= doContractAdd(txn); + terResult = doContractAdd(txn); break; case ttCONTRACT_REMOVE: - terResult=doContractRemove(txn); + terResult = doContractRemove(txn); break; default: @@ -461,14 +461,23 @@ TER TransactionEngine::applyTransaction(const SerializedTransaction& txn, Transa txnWrite(); Serializer s; - txn.add(s); - if (!mLedger->addTransaction(txID, s)) - assert(false); + if (isSetBit(params, tapOPEN_LEDGER)) + { + if (!mLedger->addTransaction(txID, s)) + assert(false); + } + else + { + Serializer m; + mNodes.calcRawMeta(m); + if (!mLedger->addTransaction(txID, s, m)) + assert(false); - // Charge whatever fee they specified. - mLedger->destroyCoins(saPaid.getNValue()); + // Charge whatever fee they specified. + mLedger->destroyCoins(saPaid.getNValue()); + } } mTxnAccount = SLE::pointer(); From 34dcb993704aa2fa1545e1032ce7d3c33478bd97 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:29:04 -0700 Subject: [PATCH 11/44] New functions to pass tree node types to callers. --- src/SHAMap.cpp | 71 ++++++++++++++++++++++++++++++++++++-------------- src/SHAMap.h | 6 +++-- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 8a232ec547..4525769dba 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -238,7 +238,7 @@ SHAMapItem::SHAMapItem(const uint256& tag, const Serializer& data) : mTag(tag), mData(data.peekData()) { ; } -SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode* node) +SHAMapTreeNode* SHAMap::firstBelow(SHAMapTreeNode* node) { // Return the first item below this node #ifdef ST_DEBUG @@ -246,7 +246,7 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode* node) #endif do { // Walk down the tree - if (node->hasItem()) return node->peekItem(); + if (node->hasItem()) return node; bool foundNode = false; for (int i = 0; i < 16; ++i) @@ -261,11 +261,12 @@ SHAMapItem::pointer SHAMap::firstBelow(SHAMapTreeNode* node) foundNode = true; break; } - if (!foundNode) return SHAMapItem::pointer(); + if (!foundNode) + return NULL; } while (true); } -SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode* node) +SHAMapTreeNode* SHAMap::lastBelow(SHAMapTreeNode* node) { #ifdef DEBUG std::cerr << "lastBelow(" << *node << ")" << std::endl; @@ -273,7 +274,8 @@ SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode* node) do { // Walk down the tree - if (node->hasItem()) return node->peekItem(); + if (node->hasItem()) + return node; bool foundNode = false; for (int i = 15; i >= 0; ++i) @@ -283,7 +285,8 @@ SHAMapItem::pointer SHAMap::lastBelow(SHAMapTreeNode* node) foundNode = true; break; } - if (!foundNode) return SHAMapItem::pointer(); + if (!foundNode) + return NULL; } while (true); } @@ -345,16 +348,39 @@ void SHAMap::eraseChildren(SHAMapTreeNode::pointer node) SHAMapItem::pointer SHAMap::peekFirstItem() { boost::recursive_mutex::scoped_lock sl(mLock); - return firstBelow(root.get()); + SHAMapTreeNode *node = firstBelow(root.get()); + if (!node) + return SHAMapItem::pointer(); + return node->peekItem(); +} + +SHAMapItem::pointer SHAMap::peekFirstItem(SHAMapTreeNode::TNType& type) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + SHAMapTreeNode *node = firstBelow(root.get()); + if (!node) + return SHAMapItem::pointer(); + type = node->getType(); + return node->peekItem(); } SHAMapItem::pointer SHAMap::peekLastItem() { boost::recursive_mutex::scoped_lock sl(mLock); - return lastBelow(root.get()); + SHAMapTreeNode *node = lastBelow(root.get()); + if (!node) + return SHAMapItem::pointer(); + return node->peekItem(); } SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) +{ + SHAMapTreeNode::TNType type; + return peekNextItem(id, type); +} + + +SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id, SHAMapTreeNode::TNType& type) { // Get a pointer to the next item in the tree after a given item - item must be in tree boost::recursive_mutex::scoped_lock sl(mLock); @@ -367,17 +393,24 @@ SHAMapItem::pointer SHAMap::peekNextItem(const uint256& id) if (node->isLeaf()) { if (node->peekItem()->getTag() > id) - return node->peekItem(); - } - else for (int i = node->selectBranch(id) + 1; i < 16; ++i) - if (!node->isEmptyBranch(i)) { - node = getNode(node->getChildNodeID(i), node->getChildHash(i), false); - SHAMapItem::pointer item = firstBelow(node.get()); - if (!item) - throw std::runtime_error("missing node"); - return item; + type = node->getType(); + return node->peekItem(); } + } + else + for (int i = node->selectBranch(id) + 1; i < 16; ++i) + if (!node->isEmptyBranch(i)) + { + SHAMapTreeNode *firstNode = getNodePointer(node->getChildNodeID(i), node->getChildHash(i)); + if (!firstNode) + throw std::runtime_error("missing node"); + firstNode = firstBelow(firstNode); + if (!firstNode) + throw std::runtime_error("missing node"); + type = firstNode->getType(); + return firstNode->peekItem(); + } } // must be last item return SHAMapItem::pointer(); @@ -402,10 +435,10 @@ SHAMapItem::pointer SHAMap::peekPrevItem(const uint256& id) if (!node->isEmptyBranch(i)) { node = getNode(node->getChildNodeID(i), node->getChildHash(i), false); - SHAMapItem::pointer item = firstBelow(node.get()); + SHAMapTreeNode* item = firstBelow(node.get()); if (!item) throw std::runtime_error("missing node"); - return item; + return item->peekItem(); } } // must be last item diff --git a/src/SHAMap.h b/src/SHAMap.h index 0f7061d71a..09ccefe738 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -280,9 +280,9 @@ protected: SHAMapTreeNode::pointer getNode(const SHAMapNode& id); SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify); SHAMapTreeNode* getNodePointer(const SHAMapNode& id, const uint256& hash); + SHAMapTreeNode* firstBelow(SHAMapTreeNode*); + SHAMapTreeNode* lastBelow(SHAMapTreeNode*); - SHAMapItem::pointer firstBelow(SHAMapTreeNode*); - SHAMapItem::pointer lastBelow(SHAMapTreeNode*); SHAMapItem::pointer onlyBelow(SHAMapTreeNode*); void eraseChildren(SHAMapTreeNode::pointer); @@ -322,8 +322,10 @@ public: // traverse functions SHAMapItem::pointer peekFirstItem(); + SHAMapItem::pointer peekFirstItem(SHAMapTreeNode::TNType& type); SHAMapItem::pointer peekLastItem(); SHAMapItem::pointer peekNextItem(const uint256&); + SHAMapItem::pointer peekNextItem(const uint256&, SHAMapTreeNode::TNType& type); SHAMapItem::pointer peekPrevItem(const uint256&); // comparison/sync functions From 4e957a5efd7146c7c7d364adb9bd6e5df9dcc31f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:29:14 -0700 Subject: [PATCH 12/44] This version is incompatible with the previous versions --- src/Version.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Version.h b/src/Version.h index df42303f00..d7cd3a02aa 100644 --- a/src/Version.h +++ b/src/Version.h @@ -5,7 +5,7 @@ // #define SERVER_VERSION_MAJOR 0 -#define SERVER_VERSION_MINOR 4 +#define SERVER_VERSION_MINOR 5 #define SERVER_VERSION_SUB "-a" #define SERVER_NAME "NewCoin" @@ -16,11 +16,11 @@ // Version we prefer to speak: #define PROTO_VERSION_MAJOR 0 -#define PROTO_VERSION_MINOR 6 +#define PROTO_VERSION_MINOR 7 // Version we will speak to: #define MIN_PROTO_MAJOR 0 -#define MIN_PROTO_MINOR 6 +#define MIN_PROTO_MINOR 7 #define MAKE_VERSION_INT(maj,min) ((maj << 16) | min) #define GET_VERSION_MAJOR(ver) (ver >> 16) From 66338e4a0aaacf6b33f2dccfe819062dbcbe1b31 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:29:27 -0700 Subject: [PATCH 13/44] Add metadata to some JSON outputs. --- src/Ledger.cpp | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index f106297d4d..2348ef63ec 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -489,14 +489,38 @@ void Ledger::addJson(Json::Value& ret, int options) if (mTransactionMap && (full || ((options & LEDGER_JSON_DUMP_TXNS) != 0))) { Json::Value txns(Json::arrayValue); - for (SHAMapItem::pointer item = mTransactionMap->peekFirstItem(); !!item; - item = mTransactionMap->peekNextItem(item->getTag())) + SHAMapTreeNode::TNType type; + for (SHAMapItem::pointer item = mTransactionMap->peekFirstItem(type); !!item; + item = mTransactionMap->peekNextItem(item->getTag(), type)) { if (full) { - SerializerIterator sit(item->peekSerializer()); - SerializedTransaction txn(sit); - txns.append(txn.getJson(0)); + if (type == SHAMapTreeNode::tnTRANSACTION_NM) + { + SerializerIterator sit(item->peekSerializer()); + SerializedTransaction txn(sit); + txns.append(txn.getJson(0)); + } + else if (type == SHAMapTreeNode::tnTRANSACTION_MD) + { + SerializerIterator sit(item->peekSerializer()); + Serializer sTxn(sit.getVL()); + Serializer sMeta(sit.getVL()); + + SerializerIterator tsit(sTxn); + SerializedTransaction txn(tsit); + + TransactionMetaSet meta(mLedgerSeq, sit.getVL()); + Json::Value txJson = txn.getJson(0); + txJson["metaData"] = meta.getJson(0); + txns.append(txJson); + } + else + { + Json::Value error = Json::objectValue; + error[item->getTag().GetHex()] = type; + txns.append(error); + } } else txns.append(item->getTag().GetHex()); } From 50190b2c35d3622c50afaf125a9865d278917238 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:31:51 -0700 Subject: [PATCH 14/44] Fix threading bug. --- src/SerializedLedger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/SerializedLedger.cpp b/src/SerializedLedger.cpp index 5796952368..41c4bc7fb8 100644 --- a/src/SerializedLedger.cpp +++ b/src/SerializedLedger.cpp @@ -102,7 +102,7 @@ bool SerializedLedgerEntry::thread(const uint256& txID, uint32 ledgerSeq, uint25 if (oldPrevTxID == txID) return false; prevTxID = oldPrevTxID; - prevLedgerID = getIFieldU32(sfLastTxnID); + prevLedgerID = getIFieldU32(sfLastTxnSeq); assert(prevTxID != txID); setIFieldH256(sfLastTxnID, txID); setIFieldU32(sfLastTxnSeq, ledgerSeq); From db85d87ff25d77e2630712e025b513846985068f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:39:03 -0700 Subject: [PATCH 15/44] Must modify nodes. --- src/LedgerEntrySet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 5c3f16f732..2e5a83c172 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -431,7 +431,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) // add any new modified nodes to the modification set for (boost::unordered_map::iterator it = newMod.begin(), end = newMod.end(); it != end; ++it) - entryCache(it->second); + entryModify(it->second); mSet.addRaw(s); } From aa35b33b198717573fb824319fe9de0dbd2309d8 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 01:39:14 -0700 Subject: [PATCH 16/44] Must calculated metadata before applying LES> --- src/TransactionEngine.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/TransactionEngine.cpp b/src/TransactionEngine.cpp index 1cf9379753..3dc45c7c15 100644 --- a/src/TransactionEngine.cpp +++ b/src/TransactionEngine.cpp @@ -15,7 +15,7 @@ void TransactionEngine::txnWrite() { - // Write back the account states and add the transaction to the ledger + // Write back the account states for (boost::unordered_map::iterator it = mNodes.begin(), end = mNodes.end(); it != end; ++it) { @@ -458,6 +458,9 @@ TER TransactionEngine::applyTransaction(const SerializedTransaction& txn, Transa if (tesSUCCESS == terResult || isTepPartial(terResult)) { // Transaction succeeded fully or (retries are not allowed and the transaction succeeded partially). + Serializer m; + mNodes.calcRawMeta(m); + txnWrite(); Serializer s; @@ -470,8 +473,6 @@ TER TransactionEngine::applyTransaction(const SerializedTransaction& txn, Transa } else { - Serializer m; - mNodes.calcRawMeta(m); if (!mLedger->addTransaction(txID, s, m)) assert(false); From 38dd9f3f88076720a863b56ddd47298e3f99ce0b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 12:29:06 -0700 Subject: [PATCH 17/44] Add metadata JSON to set JSON. Fix a bug where we try to access the existing node when we're creating a node. Extra asserts to catch original node mishandlig. --- src/LedgerEntrySet.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 2e5a83c172..0c09282c3a 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -257,6 +257,8 @@ Json::Value LedgerEntrySet::getJson(int) const } ret["nodes" ] = nodes; + ret["metaData"] = mSet.getJson(0); + return ret; } @@ -358,7 +360,8 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) continue; SLE::pointer origNode = mLedger->getSLE(it->first); - if (origNode->getType() == ltDIR_NODE) // No metadata for dir nodes + + if (origNode && (origNode->getType() == ltDIR_NODE)) // No metadata for dir nodes continue; SLE::pointer curNode = it->second.mEntry; @@ -366,6 +369,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) if (nType == TMNDeletedNode) { + assert(origNode); threadOwners(metaNode, origNode, mLedger, newMod); if (origNode->getIFieldPresent(sfAmount)) @@ -398,7 +402,10 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) } if (nType == TMNCreatedNode) // if created, thread to owner(s) + { + assert(!origNode); threadOwners(metaNode, curNode, mLedger, newMod); + } if ((nType == TMNCreatedNode) || (nType == TMNModifiedNode)) { @@ -408,6 +415,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) if (nType == TMNModifiedNode) { + assert(origNode); if (origNode->getIFieldPresent(sfAmount)) { // node has an amount, covers account root nodes and ripple nodes STAmount amount = origNode->getIValueFieldAmount(sfAmount); From 093ab0955777074cc6da223d55519a76b98d9d89 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 13:08:33 -0700 Subject: [PATCH 18/44] An empty node is legal. For example, a created node. --- src/TransactionMeta.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/TransactionMeta.cpp b/src/TransactionMeta.cpp index 8a14cbbf14..d43e55e343 100644 --- a/src/TransactionMeta.cpp +++ b/src/TransactionMeta.cpp @@ -160,11 +160,6 @@ TransactionMetaNode::TransactionMetaNode(int type, const uint256& node, Serializ void TransactionMetaNode::addRaw(Serializer& s) { - if (mEntries.empty()) - { // ack, an empty node - assert(false); - return; - } s.add8(mType); s.add256(mNode); mEntries.sort(); From 4831588c33531be2fbc714cc463edcb007fe0e14 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 13:11:29 -0700 Subject: [PATCH 19/44] Clean up JSON logging. --- src/AccountState.cpp | 6 ++---- src/SerializedObject.cpp | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/AccountState.cpp b/src/AccountState.cpp index 774e4247db..85ad825fed 100644 --- a/src/AccountState.cpp +++ b/src/AccountState.cpp @@ -9,6 +9,7 @@ #include "Ledger.h" #include "Serializer.h" +#include "Log.h" AccountState::AccountState(const NewcoinAddress& naAccountID) : mAccountID(naAccountID), mValid(false) { @@ -59,11 +60,8 @@ void AccountState::addJson(Json::Value& val) void AccountState::dump() { Json::Value j(Json::objectValue); - addJson(j); - - Json::StyledStreamWriter ssw; - ssw.write(std::cerr, j); + Log(lsINFO) << j; } // vim:ts=4 diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index 7600d803b7..af4210f3d1 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -6,6 +6,8 @@ #include "../json/writer.h" +#include "Log.h" + std::auto_ptr STObject::makeDefaultObject(SerializedTypeID id, const char *name) { switch(id) @@ -713,8 +715,7 @@ void STObject::unitTest() copy.setValueFieldU32(sfTest3, 1); if (object1.getSerializer() == copy.getSerializer()) throw std::runtime_error("STObject error"); #ifdef DEBUG - Json::StyledStreamWriter ssw; - ssw.write(std::cerr, copy.getJson(0)); + Log(lsDEBUG) << copy.getJson(0); #endif for (int i = 0; i < 1000; i++) From e6411a6afc56faba5082f461c20d14c208d1de0c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 13:11:34 -0700 Subject: [PATCH 20/44] Ensure we don't get json/value.h without json/writer.h --- src/Log.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Log.h b/src/Log.h index 42c8bb00c9..7153012226 100644 --- a/src/Log.h +++ b/src/Log.h @@ -6,6 +6,9 @@ #include #include +// Ensure that we don't get value.h without writer.h +#include "../json/json.h" + enum LogSeverity { lsTRACE = 0, From 499d9b553674c98ea755804587a325e4c4b9eb12 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 13:12:21 -0700 Subject: [PATCH 21/44] Log metadata in debug builds. --- src/LedgerEntrySet.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 0c09282c3a..305051ef37 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -441,6 +441,10 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) it != end; ++it) entryModify(it->second); +#ifdef DEBUG + Log(lsINFO) << "Metadata:" << mSet.getJson(0); +#endif + mSet.addRaw(s); } From 9cdcf50c0773a21b82a09bc9a62c3d23353b28f5 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 14:08:41 -0700 Subject: [PATCH 22/44] Mark a caution with this code. Cleanups. --- src/SHAMapDiff.cpp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index 4bef1f73d1..b425c1e7e6 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -95,13 +95,16 @@ bool SHAMap::compare(const SHAMap::pointer& otherMap, SHAMapDiff& differences, i { // compare two hash trees, add up to maxCount differences to the difference table // return value: true=complete table of differences given, false=too many differences // throws on corrupt tables or missing nodes + // CAUTION: otherMap is not locked and must be immutable std::stack nodeStack; // track nodes we've pushed - ScopedLock sl(Lock()); - if (getHash() == otherMap->getHash()) return true; - nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash())); + boost::recursive_mutex::scoped_lock sl(mLock); + if (getHash() == otherMap->getHash()) + return true; + + nodeStack.push(SHAMapDiffNode(SHAMapNode(), getHash(), otherMap->getHash())); while (!nodeStack.empty()) { SHAMapDiffNode dNode(nodeStack.top()); @@ -118,17 +121,20 @@ bool SHAMap::compare(const SHAMap::pointer& otherMap, SHAMapDiff& differences, i { differences.insert(std::make_pair(ourNode->getTag(), std::make_pair(ourNode->getItem(), otherNode->getItem()))); - if (--maxCount <= 0) return false; + if (--maxCount <= 0) + return false; } } else { differences.insert(std::make_pair(ourNode->getTag(), std::make_pair(ourNode->getItem(), SHAMapItem::pointer()))); - if (--maxCount <= 0) return false; + if (--maxCount <= 0) + return false; differences.insert(std::make_pair(otherNode->getTag(), std::make_pair(SHAMapItem::pointer(), otherNode->getItem()))); - if (--maxCount <= 0) return false; + if (--maxCount <= 0) + return false; } } else if (ourNode->isInner() && otherNode->isLeaf()) @@ -164,7 +170,8 @@ bool SHAMap::compare(const SHAMap::pointer& otherMap, SHAMapDiff& differences, i ourNode->getChildHash(i), otherNode->getChildHash(i))); } } - else assert(false); + else + assert(false); } return true; From de8288d4d59e60d7a2e1283cdcf5083a3eeab043 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:21:08 -0700 Subject: [PATCH 23/44] Cleanup transaction set sync map creation. Remove a passthrough to NetworkOPs that doesn't make much sense. --- src/LedgerConsensus.cpp | 18 ++++++++++-------- src/NetworkOPs.cpp | 6 ------ src/NetworkOPs.h | 1 - 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 515722b408..3d450295d1 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -25,8 +25,7 @@ typedef std::pair u256_lct_pair; TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) { - mMap = boost::make_shared(); - mMap->setSynching(); + mMap = boost::make_shared(hash); } void TransactionAcquire::done() @@ -34,10 +33,10 @@ void TransactionAcquire::done() if (mFailed) { Log(lsWARNING) << "Failed to acqiure TXs " << mHash; - theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); + mapComplete(mHash, SHAMap::pointer(), true); } else - theApp->getOPs().mapComplete(mHash, mMap); + mapComplete(mHash, mMap, true); } boost::weak_ptr TransactionAcquire::pmDowncast() @@ -376,7 +375,7 @@ void LedgerConsensus::takeInitialPosition(Ledger& initialLedger) } void LedgerConsensus::createDisputes(const SHAMap::pointer& m1, const SHAMap::pointer& m2) -{ +{ // m2 must be immutable SHAMap::SHAMapDiff differences; m1->compare(m2, differences, 16384); for (SHAMap::SHAMapDiff::iterator pos = differences.begin(), end = differences.end(); pos != end; ++pos) @@ -419,7 +418,8 @@ void LedgerConsensus::mapComplete(const uint256& hash, const SHAMap::pointer& ma assert((it2->first == mOurPosition->getCurrentHash()) && it2->second); createDisputes(it2->second, map); } - else assert(false); // We don't have our own position?! + else + assert(false); // We don't have our own position?! } mComplete[map->getHash()] = map; @@ -868,8 +868,10 @@ bool LedgerConsensus::peerGaveNodes(Peer::ref peer, const uint256& setHash, const std::list& nodeIDs, const std::list< std::vector >& nodeData) { boost::unordered_map::iterator acq = mAcquiring.find(setHash); - if (acq == mAcquiring.end()) return false; - return acq->second->takeNodes(nodeIDs, nodeData, peer); + if (acq == mAcquiring.end()) + return false; + return + acq->second->takeNodes(nodeIDs, nodeData, peer); } void LedgerConsensus::beginAccept() diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index cf3873ba89..1ecfdfae2e 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -712,12 +712,6 @@ bool NetworkOPs::hasTXSet(const boost::shared_ptr& peer, const uint256& se return mConsensus->peerHasSet(peer, set, status); } -void NetworkOPs::mapComplete(const uint256& hash, const SHAMap::pointer& map) -{ - if (mConsensus) - mConsensus->mapComplete(hash, map, true); -} - void NetworkOPs::endConsensus(bool correctLCL) { uint256 deadLedger = theApp->getMasterLedger().getClosedLedger()->getParentHash(); diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 04f4e731ae..e41a8dc6df 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -171,7 +171,6 @@ public: bool recvValidation(const SerializedValidation::pointer& val); SHAMap::pointer getTXMap(const uint256& hash); bool hasTXSet(const boost::shared_ptr& peer, const uint256& set, newcoin::TxSetStatus status); - void mapComplete(const uint256& hash, const SHAMap::pointer& map); // network state machine void checkState(const boost::system::error_code& result); From ec2dded961a9a2634a0458cf29a8796b2ff9d2c1 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:23:55 -0700 Subject: [PATCH 24/44] Revert "Cleanup transaction set sync map creation." This reverts commit de8288d4d59e60d7a2e1283cdcf5083a3eeab043. --- src/LedgerConsensus.cpp | 18 ++++++++---------- src/NetworkOPs.cpp | 6 ++++++ src/NetworkOPs.h | 1 + 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 3d450295d1..515722b408 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -25,7 +25,8 @@ typedef std::pair u256_lct_pair; TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) { - mMap = boost::make_shared(hash); + mMap = boost::make_shared(); + mMap->setSynching(); } void TransactionAcquire::done() @@ -33,10 +34,10 @@ void TransactionAcquire::done() if (mFailed) { Log(lsWARNING) << "Failed to acqiure TXs " << mHash; - mapComplete(mHash, SHAMap::pointer(), true); + theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); } else - mapComplete(mHash, mMap, true); + theApp->getOPs().mapComplete(mHash, mMap); } boost::weak_ptr TransactionAcquire::pmDowncast() @@ -375,7 +376,7 @@ void LedgerConsensus::takeInitialPosition(Ledger& initialLedger) } void LedgerConsensus::createDisputes(const SHAMap::pointer& m1, const SHAMap::pointer& m2) -{ // m2 must be immutable +{ SHAMap::SHAMapDiff differences; m1->compare(m2, differences, 16384); for (SHAMap::SHAMapDiff::iterator pos = differences.begin(), end = differences.end(); pos != end; ++pos) @@ -418,8 +419,7 @@ void LedgerConsensus::mapComplete(const uint256& hash, const SHAMap::pointer& ma assert((it2->first == mOurPosition->getCurrentHash()) && it2->second); createDisputes(it2->second, map); } - else - assert(false); // We don't have our own position?! + else assert(false); // We don't have our own position?! } mComplete[map->getHash()] = map; @@ -868,10 +868,8 @@ bool LedgerConsensus::peerGaveNodes(Peer::ref peer, const uint256& setHash, const std::list& nodeIDs, const std::list< std::vector >& nodeData) { boost::unordered_map::iterator acq = mAcquiring.find(setHash); - if (acq == mAcquiring.end()) - return false; - return - acq->second->takeNodes(nodeIDs, nodeData, peer); + if (acq == mAcquiring.end()) return false; + return acq->second->takeNodes(nodeIDs, nodeData, peer); } void LedgerConsensus::beginAccept() diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 1ecfdfae2e..cf3873ba89 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -712,6 +712,12 @@ bool NetworkOPs::hasTXSet(const boost::shared_ptr& peer, const uint256& se return mConsensus->peerHasSet(peer, set, status); } +void NetworkOPs::mapComplete(const uint256& hash, const SHAMap::pointer& map) +{ + if (mConsensus) + mConsensus->mapComplete(hash, map, true); +} + void NetworkOPs::endConsensus(bool correctLCL) { uint256 deadLedger = theApp->getMasterLedger().getClosedLedger()->getParentHash(); diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index e41a8dc6df..04f4e731ae 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -171,6 +171,7 @@ public: bool recvValidation(const SerializedValidation::pointer& val); SHAMap::pointer getTXMap(const uint256& hash); bool hasTXSet(const boost::shared_ptr& peer, const uint256& set, newcoin::TxSetStatus status); + void mapComplete(const uint256& hash, const SHAMap::pointer& map); // network state machine void checkState(const boost::system::error_code& result); From 7ba143b4df9d311b3aa5d2c3e53ab3dcb69075bc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:29:47 -0700 Subject: [PATCH 25/44] Revert "Cleanup transaction set sync map creation." This reverts commit de8288d4d59e60d7a2e1283cdcf5083a3eeab043. --- src/LedgerConsensus.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 515722b408..31f5812015 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -25,8 +25,7 @@ typedef std::pair u256_lct_pair; TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) { - mMap = boost::make_shared(); - mMap->setSynching(); + mMap = boost::make_shared(hash); } void TransactionAcquire::done() From f8e107a4bebf69d1486c5403b419523fb1ae2abb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:30:21 -0700 Subject: [PATCH 26/44] Extra debug. --- src/LedgerEntrySet.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 305051ef37..4dbef407b1 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -294,9 +294,13 @@ SLE::pointer LedgerEntrySet::getForMod(const uint256& node, Ledger::ref ledger, bool LedgerEntrySet::threadTx(TransactionMetaNode& metaNode, const NewcoinAddress& threadTo, Ledger::ref ledger, boost::unordered_map& newMods) { + Log(lsTRACE) << "Thread to " << threadTo.getAccountID(); SLE::pointer sle = getForMod(Ledger::getAccountRootIndex(threadTo.getAccountID()), ledger, newMods); if (!sle) + { + assert(false); return false; + } return threadTx(metaNode, sle, ledger, newMods); } From 7c7a4bd3e0670064a7318149c5e0852fc3893466 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:32:30 -0700 Subject: [PATCH 27/44] Make a field for transaction metadata. --- src/DBInit.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DBInit.cpp b/src/DBInit.cpp index e968ddf3c0..c0f7086b78 100644 --- a/src/DBInit.cpp +++ b/src/DBInit.cpp @@ -14,6 +14,7 @@ const char *TxnDBInit[] = { LedgerSeq BIGINT UNSIGNED, \ Status CHARACTER(1), \ RawTxn BLOB \ + TxnMeta BLOB \ );", "CREATE TABLE PubKeys ( \ ID CHARACTER(35) PRIMARY KEY, \ From 6c016039c2d7c55fa7809a7002d0ebea98bfe74d Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:36:56 -0700 Subject: [PATCH 28/44] Get rid of all "const SHAMap::pointer&" -> SHAMap::ref --- src/LedgerConsensus.cpp | 10 +++++----- src/LedgerConsensus.h | 10 +++++----- src/NetworkOPs.cpp | 2 +- src/NetworkOPs.h | 2 +- src/SHAMap.h | 6 ++++-- src/SHAMapDiff.cpp | 2 +- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 31f5812015..5e10b5f78f 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -374,7 +374,7 @@ void LedgerConsensus::takeInitialPosition(Ledger& initialLedger) propose(); } -void LedgerConsensus::createDisputes(const SHAMap::pointer& m1, const SHAMap::pointer& m2) +void LedgerConsensus::createDisputes(SHAMap::ref m1, SHAMap::ref m2) { SHAMap::SHAMapDiff differences; m1->compare(m2, differences, 16384); @@ -394,7 +394,7 @@ void LedgerConsensus::createDisputes(const SHAMap::pointer& m1, const SHAMap::po } } -void LedgerConsensus::mapComplete(const uint256& hash, const SHAMap::pointer& map, bool acquired) +void LedgerConsensus::mapComplete(const uint256& hash, SHAMap::ref map, bool acquired) { if (acquired) Log(lsINFO) << "We have acquired TXS " << hash; @@ -446,7 +446,7 @@ void LedgerConsensus::sendHaveTxSet(const uint256& hash, bool direct) theApp->getConnectionPool().relayMessage(NULL, packet); } -void LedgerConsensus::adjustCount(const SHAMap::pointer& map, const std::vector& peers) +void LedgerConsensus::adjustCount(SHAMap::ref map, const std::vector& peers) { // Adjust the counts on all disputed transactions based on the set of peers taking this position BOOST_FOREACH(u256_lct_pair& it, mDisputes) { @@ -954,7 +954,7 @@ void LedgerConsensus::applyTransaction(TransactionEngine& engine, const Serializ #endif } -void LedgerConsensus::applyTransactions(const SHAMap::pointer& set, Ledger::ref applyLedger, +void LedgerConsensus::applyTransactions(SHAMap::ref set, Ledger::ref applyLedger, Ledger::ref checkLedger, CanonicalTXSet& failedTransactions, bool openLgr) { TransactionEngineParams parms = openLgr ? tapOPEN_LEDGER : tapNONE; @@ -1016,7 +1016,7 @@ uint32 LedgerConsensus::roundCloseTime(uint32 closeTime) return closeTime - (closeTime % mCloseResolution); } -void LedgerConsensus::accept(const SHAMap::pointer& set) +void LedgerConsensus::accept(SHAMap::ref set) { assert(set->getHash() == mOurPosition->getCurrentHash()); diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index 17aa02691d..fcec2205a5 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -120,21 +120,21 @@ protected: // final accept logic static void Saccept(boost::shared_ptr This, SHAMap::pointer txSet); - void accept(const SHAMap::pointer& txSet); + void accept(SHAMap::ref txSet); void weHave(const uint256& id, Peer::ref avoidPeer); void startAcquiring(const TransactionAcquire::pointer&); SHAMap::pointer find(const uint256& hash); - void createDisputes(const SHAMap::pointer&, const SHAMap::pointer&); + void createDisputes(SHAMap::ref, SHAMap::ref); void addDisputedTransaction(const uint256&, const std::vector& transaction); - void adjustCount(const SHAMap::pointer& map, const std::vector& peers); + void adjustCount(SHAMap::ref map, const std::vector& peers); void propose(); void addPosition(LedgerProposal&, bool ours); void removePosition(LedgerProposal&, bool ours); void sendHaveTxSet(const uint256& set, bool direct); - void applyTransactions(const SHAMap::pointer& transactionSet, Ledger::ref targetLedger, + void applyTransactions(SHAMap::ref transactionSet, Ledger::ref targetLedger, Ledger::ref checkLedger, CanonicalTXSet& failedTransactions, bool openLgr); void applyTransaction(TransactionEngine& engine, const SerializedTransaction::pointer& txn, Ledger::ref targetLedger, CanonicalTXSet& failedTransactions, bool openLgr); @@ -162,7 +162,7 @@ public: SHAMap::pointer getTransactionTree(const uint256& hash, bool doAcquire); TransactionAcquire::pointer getAcquiring(const uint256& hash); - void mapComplete(const uint256& hash, const SHAMap::pointer& map, bool acquired); + void mapComplete(const uint256& hash, SHAMap::ref map, bool acquired); void checkLCL(); void handleLCL(const uint256& lclHash); diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index cf3873ba89..159a7a2d69 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -712,7 +712,7 @@ bool NetworkOPs::hasTXSet(const boost::shared_ptr& peer, const uint256& se return mConsensus->peerHasSet(peer, set, status); } -void NetworkOPs::mapComplete(const uint256& hash, const SHAMap::pointer& map) +void NetworkOPs::mapComplete(const uint256& hash, SHAMap::ref map) { if (mConsensus) mConsensus->mapComplete(hash, map, true); diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 04f4e731ae..970a27c633 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -171,7 +171,7 @@ public: bool recvValidation(const SerializedValidation::pointer& val); SHAMap::pointer getTXMap(const uint256& hash); bool hasTXSet(const boost::shared_ptr& peer, const uint256& set, newcoin::TxSetStatus status); - void mapComplete(const uint256& hash, const SHAMap::pointer& map); + void mapComplete(const uint256& hash, SHAMap::ref& map); // network state machine void checkState(const boost::system::error_code& result); diff --git a/src/SHAMap.h b/src/SHAMap.h index 09ccefe738..08712611f7 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -24,7 +24,8 @@ class SHAMap; class SHAMapNode { // Identifies a node in a SHA256 hash public: - typedef boost::shared_ptr pointer; + typedef boost::shared_ptr pointer; + typedef const boost::shared_ptr& ref; private: static uint256 smMasks[65]; // AND with hash to get node id @@ -255,6 +256,7 @@ class SHAMap { public: typedef boost::shared_ptr pointer; + typedef const boost::shared_ptr& ref; typedef std::map > SHAMapDiff; private: @@ -350,7 +352,7 @@ public: // caution: otherMap must be accessed only by this function // return value: true=successfully completed, false=too different - bool compare(const SHAMap::pointer& otherMap, SHAMapDiff& differences, int maxCount); + bool compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount); void armDirty(); int flushDirty(int maxNodes, HashedObjectType t, uint32 seq); diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index b425c1e7e6..9c147ac80e 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -91,7 +91,7 @@ bool SHAMap::walkBranch(SHAMapTreeNode* node, SHAMapItem::pointer otherMapItem, return true; } -bool SHAMap::compare(const SHAMap::pointer& otherMap, SHAMapDiff& differences, int maxCount) +bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount) { // compare two hash trees, add up to maxCount differences to the difference table // return value: true=complete table of differences given, false=too many differences // throws on corrupt tables or missing nodes From 58befb406e7aa2b12dcaaf6c7b78b1591d6bd74d Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 15:37:48 -0700 Subject: [PATCH 29/44] Typo. --- src/NetworkOPs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 970a27c633..ed6e7f0471 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -171,7 +171,7 @@ public: bool recvValidation(const SerializedValidation::pointer& val); SHAMap::pointer getTXMap(const uint256& hash); bool hasTXSet(const boost::shared_ptr& peer, const uint256& set, newcoin::TxSetStatus status); - void mapComplete(const uint256& hash, SHAMap::ref& map); + void mapComplete(const uint256& hash, SHAMap::ref map); // network state machine void checkState(const boost::system::error_code& result); From 1faa8ccda6eb0d1dedac64a93d19a87f47e42cf4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 19:26:40 -0700 Subject: [PATCH 30/44] Some less confusing names. --- src/LedgerConsensus.cpp | 69 ++++++++++++++++++++++------------------- src/LedgerConsensus.h | 12 +++---- 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 5e10b5f78f..f7fa15de13 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -173,18 +173,18 @@ void LCTransaction::unVote(const uint160& peer) } } -bool LCTransaction::updatePosition(int percentTime, bool proposing) +bool LCTransaction::updateVote(int percentTime, bool proposing) { - if (mOurPosition && (mNays == 0)) + if (mOurVote && (mNays == 0)) return false; - if (!mOurPosition && (mYays == 0)) + if (!mOurVote && (mYays == 0)) return false; bool newPosition; if (proposing) // give ourselves full weight { // This is basically the percentage of nodes voting 'yes' (including us) - int weight = (mYays * 100 + (mOurPosition ? 100 : 0)) / (mNays + mYays + 1); + int weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); // To prevent avalanche stalls, we increase the needed weight slightly over time if (percentTime < AV_MID_CONSENSUS_TIME) newPosition = weight > AV_INIT_CONSENSUS_PCT; @@ -194,16 +194,16 @@ bool LCTransaction::updatePosition(int percentTime, bool proposing) else // don't let us outweight a proposing node, just recognize consensus newPosition = mYays > mNays; - if (newPosition == mOurPosition) + if (newPosition == mOurVote) { #ifdef LC_DEBUG - Log(lsTRACE) << "No change (" << (mOurPosition ? "YES" : "NO") << ") : weight " + Log(lsTRACE) << "No change (" << (mOurVote ? "YES" : "NO") << ") : weight " << weight << ", percent " << percentTime; #endif return false; } - mOurPosition = newPosition; - Log(lsTRACE) << "We now vote " << (mOurPosition ? "YES" : "NO") << " on " << mTransactionID; + mOurVote = newPosition; + Log(lsTRACE) << "We now vote " << (mOurVote ? "YES" : "NO") << " on " << mTransactionID; return true; } @@ -357,9 +357,9 @@ void LedgerConsensus::takeInitialPosition(Ledger& initialLedger) { uint256 set = it.second->getCurrentHash(); if (found.insert(set).second) - { - boost::unordered_map::iterator iit = mComplete.find(set); - if (iit != mComplete.end()) + { // OPTIMIZEME: Don't process the same set more than once + boost::unordered_map::iterator iit = mAcquired.find(set); + if (iit != mAcquired.end()) createDisputes(initialSet, iit->second); } } @@ -402,25 +402,27 @@ void LedgerConsensus::mapComplete(const uint256& hash, SHAMap::ref map, bool acq if (!map) { // this is an invalid/corrupt map - mComplete[hash] = map; + mAcquired[hash] = map; Log(lsWARNING) << "A trusted node directed us to acquire an invalid TXN map"; return; } + assert(hash == map->getHash()); - if (mComplete.find(hash) != mComplete.end()) + if (mAcquired.find(hash) != mAcquired.end()) return; // we already have this map - if (mOurPosition && (map->getHash() != mOurPosition->getCurrentHash())) + if (mOurPosition && (hash != mOurPosition->getCurrentHash())) { // this could create disputed transactions - boost::unordered_map::iterator it2 = mComplete.find(mOurPosition->getCurrentHash()); - if (it2 != mComplete.end()) + boost::unordered_map::iterator it2 = mAcquired.find(mOurPosition->getCurrentHash()); + if (it2 != mAcquired.end()) { assert((it2->first == mOurPosition->getCurrentHash()) && it2->second); createDisputes(it2->second, map); } - else assert(false); // We don't have our own position?! + else + assert(false); // We don't have our own position?! } - mComplete[map->getHash()] = map; + mAcquired[hash] = map; // Adjust tracking for each peer that takes this position std::vector peers; @@ -594,14 +596,15 @@ void LedgerConsensus::updateOurPositions() BOOST_FOREACH(u256_lct_pair& it, mDisputes) { - if (it.second->updatePosition(mClosePercent, mProposing)) + if (it.second->updateVote(mClosePercent, mProposing)) { if (!changes) { - ourPosition = mComplete[mOurPosition->getCurrentHash()]->snapShot(true); + ourPosition = mAcquired[mOurPosition->getCurrentHash()]->snapShot(true); + assert(ourPosition); changes = true; } - if (it.second->getOurPosition()) // now a yes + if (it.second->getOurVote()) // now a yes { ourPosition->addItem(SHAMapItem(it.first, it.second->peekTransaction()), true, false); // addedTx.push_back(it.first); @@ -662,18 +665,19 @@ void LedgerConsensus::updateOurPositions() ((closeTime != (roundCloseTime(mOurPosition->getCloseTime()))) || (mOurPosition->isStale(ourCutoff)))) { // close time changed or our position is stale - ourPosition = mComplete[mOurPosition->getCurrentHash()]->snapShot(true); + ourPosition = mAcquired[mOurPosition->getCurrentHash()]->snapShot(true); + assert(ourPosition); changes = true; } if (changes) { uint256 newHash = ourPosition->getHash(); + Log(lsINFO) << "Position change: CTime " << closeTime << ", tx " << newHash; mOurPosition->changePosition(newHash, closeTime); if (mProposing) propose(); mapComplete(newHash, ourPosition, false); - Log(lsINFO) << "Position change: CTime " << closeTime << ", tx " << newHash; } } @@ -695,8 +699,8 @@ bool LedgerConsensus::haveConsensus() SHAMap::pointer LedgerConsensus::getTransactionTree(const uint256& hash, bool doAcquire) { - boost::unordered_map::iterator it = mComplete.find(hash); - if (it == mComplete.end()) + boost::unordered_map::iterator it = mAcquired.find(hash); + if (it == mAcquired.end()) { // we have not completed acquiring this ledger if (mState == lcsPRE_CLOSE) @@ -780,10 +784,11 @@ void LedgerConsensus::addDisputedTransaction(const uint256& txID, const std::vec bool ourPosition = false; if (mOurPosition) { - boost::unordered_map::iterator mit = mComplete.find(mOurPosition->getCurrentHash()); - if (mit != mComplete.end()) + boost::unordered_map::iterator mit = mAcquired.find(mOurPosition->getCurrentHash()); + if (mit != mAcquired.end()) ourPosition = mit->second->hasItem(txID); - else assert(false); // We don't have our own position? + else + assert(false); // We don't have our own position? } LCTransaction::pointer txn = boost::make_shared(txID, tx, ourPosition); @@ -792,8 +797,8 @@ void LedgerConsensus::addDisputedTransaction(const uint256& txID, const std::vec BOOST_FOREACH(u160_prop_pair& pit, mPeerPositions) { boost::unordered_map::const_iterator cit = - mComplete.find(pit.second->getCurrentHash()); - if (cit != mComplete.end() && cit->second) + mAcquired.find(pit.second->getCurrentHash()); + if (cit != mAcquired.end() && cit->second) txn->setVote(pit.first, cit->second->hasItem(txID)); } } @@ -873,7 +878,7 @@ bool LedgerConsensus::peerGaveNodes(Peer::ref peer, const uint256& setHash, void LedgerConsensus::beginAccept() { - SHAMap::pointer consensusSet = mComplete[mOurPosition->getCurrentHash()]; + SHAMap::pointer consensusSet = mAcquired[mOurPosition->getCurrentHash()]; if (!consensusSet) { Log(lsFATAL) << "We don't have a consensus set"; @@ -1070,7 +1075,7 @@ void LedgerConsensus::accept(SHAMap::ref set) TransactionEngine engine(newOL); BOOST_FOREACH(u256_lct_pair& it, mDisputes) { - if (!it.second->getOurPosition()) + if (!it.second->getOurVote()) { // we voted NO try { diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index fcec2205a5..a24a57565a 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -49,24 +49,24 @@ class LCTransaction protected: uint256 mTransactionID; int mYays, mNays; - bool mOurPosition; + bool mOurVote; Serializer transaction; boost::unordered_map mVotes; public: typedef boost::shared_ptr pointer; - LCTransaction(const uint256 &txID, const std::vector& tx, bool ourPosition) : - mTransactionID(txID), mYays(0), mNays(0), mOurPosition(ourPosition), transaction(tx) { ; } + LCTransaction(const uint256 &txID, const std::vector& tx, bool ourVote) : + mTransactionID(txID), mYays(0), mNays(0), mOurVote(ourVote), transaction(tx) { ; } const uint256& getTransactionID() const { return mTransactionID; } - bool getOurPosition() const { return mOurPosition; } + bool getOurVote() const { return mOurVote; } Serializer& peekTransaction() { return transaction; } void setVote(const uint160& peer, bool votesYes); void unVote(const uint160& peer); - bool updatePosition(int percentTime, bool proposing); + bool updateVote(int percentTime, bool proposing); }; enum LCState @@ -100,7 +100,7 @@ protected: boost::unordered_map mPeerPositions; // Transaction Sets, indexed by hash of transaction tree - boost::unordered_map mComplete; + boost::unordered_map mAcquired; boost::unordered_map mAcquiring; // Peer sets From b35f87564a737e20cb68e6c8b289022a6e6bf24c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 19:32:44 -0700 Subject: [PATCH 31/44] Fix a bug triggered by a acquiring a transaction set after having bowed out of the consensus process. --- src/LedgerConsensus.cpp | 12 +++++++----- src/LedgerProposal.cpp | 7 +++++-- src/LedgerProposal.h | 3 ++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index f7fa15de13..71dca921ca 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -411,7 +411,7 @@ void LedgerConsensus::mapComplete(const uint256& hash, SHAMap::ref map, bool acq if (mAcquired.find(hash) != mAcquired.end()) return; // we already have this map - if (mOurPosition && (hash != mOurPosition->getCurrentHash())) + if (mOurPosition && (!mOurPosition->isBowOut()) && (hash != mOurPosition->getCurrentHash())) { // this could create disputed transactions boost::unordered_map::iterator it2 = mAcquired.find(mOurPosition->getCurrentHash()); if (it2 != mAcquired.end()) @@ -674,10 +674,12 @@ void LedgerConsensus::updateOurPositions() { uint256 newHash = ourPosition->getHash(); Log(lsINFO) << "Position change: CTime " << closeTime << ", tx " << newHash; - mOurPosition->changePosition(newHash, closeTime); - if (mProposing) - propose(); - mapComplete(newHash, ourPosition, false); + if (mOurPosition->changePosition(newHash, closeTime)) + { + if (mProposing) + propose(); + mapComplete(newHash, ourPosition, false); + } } } diff --git a/src/LedgerProposal.cpp b/src/LedgerProposal.cpp index 5840c10372..a334a612f7 100644 --- a/src/LedgerProposal.cpp +++ b/src/LedgerProposal.cpp @@ -54,17 +54,20 @@ bool LedgerProposal::checkSign(const std::string& signature, const uint256& sign return mPublicKey.verifyNodePublic(signingHash, signature); } -void LedgerProposal::changePosition(const uint256& newPosition, uint32 closeTime) +bool LedgerProposal::changePosition(const uint256& newPosition, uint32 closeTime) { + if (mProposeSeq == seqLeave) + return false; + mCurrentHash = newPosition; mCloseTime = closeTime; mTime = boost::posix_time::second_clock::universal_time(); ++mProposeSeq; + return true; } void LedgerProposal::bowOut() { - mCurrentHash = uint256(); mTime = boost::posix_time::second_clock::universal_time(); mProposeSeq = seqLeave; } diff --git a/src/LedgerProposal.h b/src/LedgerProposal.h index adaa557719..fb7df34309 100644 --- a/src/LedgerProposal.h +++ b/src/LedgerProposal.h @@ -59,11 +59,12 @@ public: void setSignature(const std::string& signature) { mSignature = signature; } bool hasSignature() { return !mSignature.empty(); } bool isPrevLedger(const uint256& pl) { return mPreviousLedger == pl; } + bool isBowOut() { return mProposeSeq == seqLeave; } const boost::posix_time::ptime getCreateTime() { return mTime; } bool isStale(boost::posix_time::ptime cutoff) { return mTime <= cutoff; } - void changePosition(const uint256& newPosition, uint32 newCloseTime); + bool changePosition(const uint256& newPosition, uint32 newCloseTime); void bowOut(); Json::Value getJson() const; }; From 0efe8b48929769038d0b1f9141fad2536ff543cf Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 20:41:25 -0700 Subject: [PATCH 32/44] Cleanups. --- src/LedgerAcquire.cpp | 5 ++--- src/LedgerAcquire.h | 1 + src/LedgerConsensus.cpp | 8 +++++--- src/LedgerConsensus.h | 4 ++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 353b7d8872..45c5ff6d4f 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -84,9 +84,8 @@ void PeerSet::TimerEntry(boost::weak_ptr wptr, const boost::system::err if (result == boost::asio::error::operation_aborted) return; boost::shared_ptr ptr = wptr.lock(); - if (!ptr) - return; - ptr->invokeOnTimer(); + if (ptr) + ptr->invokeOnTimer(); } LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT), diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index 6781eddf88..b589618299 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -78,6 +78,7 @@ protected: public: LedgerAcquire(const uint256& hash); + virtual ~LedgerAcquire() { ; } bool isBase() const { return mHaveBase; } bool isAcctStComplete() const { return mHaveState; } diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 71dca921ca..70895ff4f0 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -32,7 +32,7 @@ void TransactionAcquire::done() { if (mFailed) { - Log(lsWARNING) << "Failed to acqiure TXs " << mHash; + Log(lsWARNING) << "Failed to acquire TXs " << mHash; theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); } else @@ -108,7 +108,8 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, } if (!mMap->addRootNode(getHash(), *nodeDatait, snfWIRE)) return false; - else mHaveRoot = true; + else + mHaveRoot = true; } else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) return false; @@ -874,7 +875,8 @@ bool LedgerConsensus::peerGaveNodes(Peer::ref peer, const uint256& setHash, const std::list& nodeIDs, const std::list< std::vector >& nodeData) { boost::unordered_map::iterator acq = mAcquiring.find(setHash); - if (acq == mAcquiring.end()) return false; + if (acq == mAcquiring.end()) + return false; return acq->second->takeNodes(nodeIDs, nodeData, peer); } diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index a24a57565a..3833c300b3 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -37,11 +37,11 @@ protected: public: TransactionAcquire(const uint256& hash); + virtual ~TransactionAcquire() { ; } SHAMap::pointer getMap() { return mMap; } - bool takeNodes(const std::list& IDs, const std::list< std::vector >& data, - Peer::ref); + bool takeNodes(const std::list& IDs, const std::list< std::vector >& data, Peer::ref); }; class LCTransaction From bbddef85726fb49fd5ee2cda4d85c3bd62376294 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 20:49:36 -0700 Subject: [PATCH 33/44] Throw on missing node in SHAMapDiff code. --- src/SHAMapDiff.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index 9c147ac80e..0eecbb95ec 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -112,6 +112,11 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount SHAMapTreeNode* ourNode = getNodePointer(dNode.mNodeID, dNode.mOurHash); SHAMapTreeNode* otherNode = otherMap->getNodePointer(dNode.mNodeID, dNode.mOtherHash); + if (!ourNode || !otherNode) + { + assert(false); + throw SHAMapMissingNode(dNode.mNodeID, uint256()); + } if (ourNode->isLeaf() && otherNode->isLeaf()) { // two leaves From f7e68cfc51abb694e1180ed06c5c79df41057038 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Sep 2012 20:49:48 -0700 Subject: [PATCH 34/44] Fix SHAMap state when tx sync completes. --- src/LedgerConsensus.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 70895ff4f0..d507d9fee6 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -36,7 +36,10 @@ void TransactionAcquire::done() theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); } else + { + mMap->setImmutable(); theApp->getOPs().mapComplete(mHash, mMap); + } } boost::weak_ptr TransactionAcquire::pmDowncast() From 205f5e4a632cb29d6d4a84f6c2dafa56f157717c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 07:07:44 -0700 Subject: [PATCH 35/44] Call to get the auxiliary I/O service --- src/Application.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Application.h b/src/Application.h index 79ffb8cf34..6511d0f566 100644 --- a/src/Application.h +++ b/src/Application.h @@ -78,6 +78,7 @@ public: NetworkOPs& getOPs() { return mNetOps; } boost::asio::io_service& getIOService() { return mIOService; } + boost::asio::io_service& getAuxService() { return mAuxService; } LedgerMaster& getMasterLedger() { return mMasterLedger; } LedgerAcquireMaster& getMasterLedgerAcquire() { return mMasterLedgerAcquire; } From a4f41edf7a579694e7b0d60a5c5ffbd548f7d299 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 07:08:01 -0700 Subject: [PATCH 36/44] Cleanup shared polymorphic downcast. --- src/LedgerAcquire.cpp | 2 +- src/LedgerConsensus.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 45c5ff6d4f..bbfd67063b 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -108,7 +108,7 @@ void LedgerAcquire::onTimer() boost::weak_ptr LedgerAcquire::pmDowncast() { - return boost::shared_polymorphic_downcast(shared_from_this()); + return boost::shared_polymorphic_downcast(shared_from_this()); } void LedgerAcquire::done() diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index d507d9fee6..f223f7742b 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -44,7 +44,7 @@ void TransactionAcquire::done() boost::weak_ptr TransactionAcquire::pmDowncast() { - return boost::shared_polymorphic_downcast(shared_from_this()); + return boost::shared_polymorphic_downcast(shared_from_this()); } void TransactionAcquire::trigger(Peer::ref peer, bool timer) From 72f94171490eb45d98066feb40d315ac2065085a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 07:08:16 -0700 Subject: [PATCH 37/44] Add an assert. --- src/SHAMapDiff.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index 0eecbb95ec..3f5adb3e0d 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -97,6 +97,8 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount // throws on corrupt tables or missing nodes // CAUTION: otherMap is not locked and must be immutable + assert(isValid() && otherMap && otherMap->isValid()); + std::stack nodeStack; // track nodes we've pushed boost::recursive_mutex::scoped_lock sl(mLock); From 72b2478a7b5dea5fbf40d512b86e322dfca90366 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 07:08:27 -0700 Subject: [PATCH 38/44] Rename SHAMap states. --- src/SHAMap.cpp | 14 +++++++------- src/SHAMap.h | 30 ++++++++++++++++++------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 4525769dba..652326f177 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -39,14 +39,14 @@ std::size_t hash_value(const uint160& u) } -SHAMap::SHAMap(uint32 seq) : mSeq(seq), mState(Modifying) +SHAMap::SHAMap(uint32 seq) : mSeq(seq), mState(smsModifying) { root = boost::make_shared(mSeq, SHAMapNode(0, uint256())); root->makeInner(); mTNByID[*root] = root; } -SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(Synching) +SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(smsSynching) { // FIXME: Need to acquire root node root = boost::make_shared(mSeq, SHAMapNode(0, uint256())); root->makeInner(); @@ -62,7 +62,7 @@ SHAMap::pointer SHAMap::snapShot(bool isMutable) newMap.mTNByID = mTNByID; newMap.root = root; if (!isMutable) - newMap.mState = Immutable; + newMap.mState = smsImmutable; return ret; } @@ -105,7 +105,7 @@ void SHAMap::dirtyUp(std::stack& stack, const uint256& { // walk the tree up from through the inner nodes to the root // update linking hashes and add nodes to dirty list - assert((mState != Synching) && (mState != Immutable)); + assert((mState != smsSynching) && (mState != smsImmutable)); while (!stack.empty()) { @@ -475,7 +475,7 @@ bool SHAMap::hasItem(const uint256& id) bool SHAMap::delItem(const uint256& id) { // delete the item with this ID boost::recursive_mutex::scoped_lock sl(mLock); - assert(mState != Immutable); + assert(mState != smsImmutable); std::stack stack = getStack(id, true, false); if (stack.empty()) @@ -552,7 +552,7 @@ bool SHAMap::addGiveItem(const SHAMapItem::pointer& item, bool isTransaction, bo (hasMeta ? SHAMapTreeNode::tnTRANSACTION_MD : SHAMapTreeNode::tnTRANSACTION_NM); boost::recursive_mutex::scoped_lock sl(mLock); - assert(mState != Immutable); + assert(mState != smsImmutable); std::stack stack = getStack(tag, true, false); if (stack.empty()) @@ -644,7 +644,7 @@ bool SHAMap::updateGiveItem(const SHAMapItem::pointer& item, bool isTransaction, uint256 tag = item->getTag(); boost::recursive_mutex::scoped_lock sl(mLock); - assert(mState != Immutable); + assert(mState != smsImmutable); std::stack stack = getStack(tag, true, false); if (stack.empty()) throw std::runtime_error("missing node"); diff --git a/src/SHAMap.h b/src/SHAMap.h index 08712611f7..e308cc800e 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -217,11 +217,11 @@ public: enum SHAMapState { - Modifying = 0, // Objects can be added and removed (like an open ledger) - Immutable = 1, // Map cannot be changed (like a closed ledger) - Synching = 2, // Map's hash is locked in, valid nodes can be added (like a peer's closing ledger) - Floating = 3, // Map is free to change hash (like a synching open ledger) - Invalid = 4, // Map is known not to be valid (usually synching a corrupt ledger) + smsModifying = 0, // Objects can be added and removed (like an open ledger) + smsImmutable = 1, // Map cannot be changed (like a closed ledger) + smsSynching = 2, // Map's hash is locked in, valid nodes can be added (like a peer's closing ledger) + smsFloating = 3, // Map is free to change hash (like a synching open ledger) + smsInvalid = 4, // Map is known not to be valid (usually synching a corrupt ledger) }; class SHAMapSyncFilter @@ -229,9 +229,11 @@ class SHAMapSyncFilter public: SHAMapSyncFilter() { ; } virtual ~SHAMapSyncFilter() { ; } + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, bool isLeaf) { ; } + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { return false; } }; @@ -248,6 +250,7 @@ public: { ; } virtual ~SHAMapMissingNode() throw() { ; } + const SHAMapNode& getNodeID() const { return mNodeID; } const uint256& getNodeHash() const { return mNodeHash; } }; @@ -257,6 +260,7 @@ class SHAMap public: typedef boost::shared_ptr pointer; typedef const boost::shared_ptr& ref; + typedef std::map > SHAMapDiff; private: @@ -297,6 +301,8 @@ public: SHAMap(uint32 seq = 0); SHAMap(const uint256& hash); + ~SHAMap() { mState = smsInvalid; } + // Returns a new map that's a snapshot of this one. Force CoW SHAMap::pointer snapShot(bool isMutable); @@ -342,13 +348,13 @@ public: SHAMapSyncFilter* filter); // status functions - void setImmutable(void) { assert(mState != Invalid); mState = Immutable; } - void clearImmutable(void) { mState = Modifying; } - bool isSynching(void) const { return (mState == Floating) || (mState == Synching); } - void setSynching(void) { mState = Synching; } - void setFloating(void) { mState = Floating; } - void clearSynching(void) { mState = Modifying; } - bool isValid(void) { return mState != Invalid; } + void setImmutable(void) { assert(mState != smsInvalid); mState = smsImmutable; } + void clearImmutable(void) { mState = smsModifying; } + bool isSynching(void) const { return (mState == smsFloating) || (mState == smsSynching); } + void setSynching(void) { mState = smsSynching; } + void setFloating(void) { mState = smsFloating; } + void clearSynching(void) { mState = smsModifying; } + bool isValid(void) { return mState != smsInvalid; } // caution: otherMap must be accessed only by this function // return value: true=successfully completed, false=too different From c9a44e4a1af8b03ab6a74904a21440dba0e0eb76 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 08:40:55 -0700 Subject: [PATCH 39/44] We have to make sure someone holds a strong pointer to the acquiring set when we move it from acquring to acquired. --- src/LedgerConsensus.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index f223f7742b..652b70265d 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -880,7 +880,8 @@ bool LedgerConsensus::peerGaveNodes(Peer::ref peer, const uint256& setHash, boost::unordered_map::iterator acq = mAcquiring.find(setHash); if (acq == mAcquiring.end()) return false; - return acq->second->takeNodes(nodeIDs, nodeData, peer); + TransactionAcquire::pointer set = acq->second; // We must keep the set around during the function + return set->takeNodes(nodeIDs, nodeData, peer); } void LedgerConsensus::beginAccept() From faece188f4d37aa51f4658ff251f4b3ff880a29f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 08:41:59 -0700 Subject: [PATCH 40/44] Belt and suspenders. Fix on both sides. --- src/LedgerConsensus.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 652b70265d..066c22e6e6 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -402,11 +402,11 @@ void LedgerConsensus::mapComplete(const uint256& hash, SHAMap::ref map, bool acq { if (acquired) Log(lsINFO) << "We have acquired TXS " << hash; - mAcquiring.erase(hash); if (!map) { // this is an invalid/corrupt map mAcquired[hash] = map; + mAcquiring.erase(hash); Log(lsWARNING) << "A trusted node directed us to acquire an invalid TXN map"; return; } @@ -427,6 +427,7 @@ void LedgerConsensus::mapComplete(const uint256& hash, SHAMap::ref map, bool acq assert(false); // We don't have our own position?! } mAcquired[hash] = map; + mAcquiring.erase(hash); // Adjust tracking for each peer that takes this position std::vector peers; From 3a786b911c17d50bd5cf93870ae40d58fb4c8382 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 08:42:21 -0700 Subject: [PATCH 41/44] Don't crash if we can't find the ledger for the generator --- src/NetworkOPs.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 159a7a2d69..0433892263 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -211,7 +211,11 @@ SLE::pointer NetworkOPs::getGenerator(const uint256& uLedger, const uint160& uGe { LedgerStateParms qry = lepNONE; - return mLedgerMaster->getLedgerByHash(uLedger)->getGenerator(qry, uGeneratorID); + Ledger::pointer ledger = mLedgerMaster->getLedgerByHash(uLedger); + if (!ledger) + return SLE::pointer(); + else + return ledger->getGenerator(qry, uGeneratorID); } // From b69a0b14bf507ef7079adfb4ec22a10e2d71bf4f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 08:42:39 -0700 Subject: [PATCH 42/44] Style change. --- src/SHAMap.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/SHAMap.h b/src/SHAMap.h index e308cc800e..e7f5046d5c 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -348,13 +348,13 @@ public: SHAMapSyncFilter* filter); // status functions - void setImmutable(void) { assert(mState != smsInvalid); mState = smsImmutable; } - void clearImmutable(void) { mState = smsModifying; } - bool isSynching(void) const { return (mState == smsFloating) || (mState == smsSynching); } - void setSynching(void) { mState = smsSynching; } - void setFloating(void) { mState = smsFloating; } - void clearSynching(void) { mState = smsModifying; } - bool isValid(void) { return mState != smsInvalid; } + void setImmutable() { assert(mState != smsInvalid); mState = smsImmutable; } + void clearImmutable() { mState = smsModifying; } + bool isSynching() const { return (mState == smsFloating) || (mState == smsSynching); } + void setSynching() { mState = smsSynching; } + void setFloating() { mState = smsFloating; } + void clearSynching() { mState = smsModifying; } + bool isValid() { return mState != smsInvalid; } // caution: otherMap must be accessed only by this function // return value: true=successfully completed, false=too different From 22f9a1a2582e4f7f2945f62abd916ecb9be2ae78 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 09:58:51 -0700 Subject: [PATCH 43/44] Don't let bow outs count as agreeing/disagreeing for consensus determination --- src/LedgerConsensus.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 066c22e6e6..9cfa4fd23a 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -185,18 +185,22 @@ bool LCTransaction::updateVote(int percentTime, bool proposing) return false; bool newPosition; + int weight; if (proposing) // give ourselves full weight { // This is basically the percentage of nodes voting 'yes' (including us) - int weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); + weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); // To prevent avalanche stalls, we increase the needed weight slightly over time if (percentTime < AV_MID_CONSENSUS_TIME) newPosition = weight > AV_INIT_CONSENSUS_PCT; else if (percentTime < AV_LATE_CONSENSUS_TIME) newPosition = weight > AV_MID_CONSENSUS_PCT; else newPosition = weight > AV_LATE_CONSENSUS_PCT; } - else // don't let us outweight a proposing node, just recognize consensus + else // don't let us outweigh a proposing node, just recognize consensus + { + weight = -1; newPosition = mYays > mNays; + } if (newPosition == mOurVote) { @@ -689,17 +693,26 @@ void LedgerConsensus::updateOurPositions() } bool LedgerConsensus::haveConsensus() -{ +{ // FIXME: Should check for a supermajority on each disputed transaction + // counting unacquired TX sets as disagreeing int agree = 0, disagree = 0; uint256 ourPosition = mOurPosition->getCurrentHash(); BOOST_FOREACH(u160_prop_pair& it, mPeerPositions) { - if (it.second->getCurrentHash() == ourPosition) - ++agree; - else - ++disagree; + if (!it.second->isBowOut()) + { + if (it.second->getCurrentHash() == ourPosition) + ++agree; + else + ++disagree; + } } int currentValidations = theApp->getValidations().getNodesAfter(mPrevLedgerHash); + +#ifdef LC_DEBUG + Log(lsINFO) << "Checking for TX consensus: agree=" << agree << ", disagree=" << disagree; +#endif + return ContinuousLedgerTiming::haveConsensus(mPreviousProposers, agree + disagree, agree, currentValidations, mPreviousMSeconds, mCurrentMSeconds); } From d147b61530a6019ad7cb7adb751b330d2a9a857e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Sep 2012 10:06:23 -0700 Subject: [PATCH 44/44] The ledger accept process calls functions like ConnectionPool::relayMessage and so must be called in the main I/O thread, at least for now. --- src/LedgerConsensus.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 9cfa4fd23a..aff9f7bdaf 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -909,8 +909,7 @@ void LedgerConsensus::beginAccept() } theApp->getOPs().newLCL(mPeerPositions.size(), mCurrentMSeconds, mNewLedgerHash); - boost::thread thread(boost::bind(&LedgerConsensus::Saccept, shared_from_this(), consensusSet)); - thread.detach(); + theApp->getIOService().post(boost::bind(&LedgerConsensus::Saccept, shared_from_this(), consensusSet)); } void LedgerConsensus::Saccept(boost::shared_ptr This, SHAMap::pointer txSet)