From 94a9def72e3788c40091b2a3c11c664df73cb692 Mon Sep 17 00:00:00 2001 From: jed Date: Fri, 29 Jun 2012 21:36:29 -0700 Subject: [PATCH 01/14] tweak json returns --- src/NetworkOPs.cpp | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 06590e3ca2..c29da13f69 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -769,13 +769,6 @@ void NetworkOPs::pubLedger(const Ledger::pointer& lpAccepted) Json::Value NetworkOPs::transJson(const SerializedTransaction& stTxn, TransactionEngineResult terResult, const std::string& strStatus, int iSeq, const std::string& strType) { - Json::Value jvAccounts(Json::arrayValue); - - BOOST_FOREACH(const NewcoinAddress& naAccountID, stTxn.getAffectedAccounts()) - { - jvAccounts.append(Json::Value(naAccountID.humanAccountID())); - } - Json::Value jvObj(Json::objectValue); std::string strToken; std::string strHuman; @@ -783,12 +776,10 @@ Json::Value NetworkOPs::transJson(const SerializedTransaction& stTxn, Transactio transResultInfo(terResult, strToken, strHuman); jvObj["type"] = strType; - jvObj["seq"] = iSeq; - jvObj["accounts"] = jvAccounts; jvObj["transaction"] = stTxn.getJson(0); - jvObj["status"] = strStatus; + jvObj["transaction"]["inLedger"] = iSeq; + jvObj["transaction"]["status"] = strStatus; jvObj["result"] = strToken; - jvObj["result_message"] = strHuman; jvObj["result_code"] = terResult; return jvObj; From 6bbf7f58f025142522e303d66a2c5e9a29ccd426 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 22:40:48 -0700 Subject: [PATCH 02/14] Fix a small bug. Catch bad store/retrieeves sooner. --- src/HashedObject.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 4305a5b5cb..8c9ef0ac7c 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -17,6 +17,7 @@ HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false=already in cache, true = added to cache + assert(hash == Serializer::getSHA512Half(data)); if (!theApp->getHashNodeDB()) return true; if (mCache.touch(hash)) { @@ -25,7 +26,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, } HashedObject::pointer object = boost::make_shared(type, index, data, hash); - + if (!mCache.canonicalize(hash, object)) { boost::recursive_mutex::scoped_lock sl(mWriteMutex); mWriteSet.push_back(object); @@ -126,6 +127,8 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) db->getBinary("Object", &(data.front()), size); db->endIterRows(); + assert(Serializer::getSHA512Half(data) == hash); + HashedObjectType htype = UNKNOWN; switch(type[0]) { From b95dc249c06ac7a2d3d4704ee765446aefa9bc58 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 22:41:14 -0700 Subject: [PATCH 03/14] Fix two bugs that could have been what Jed reported. --- src/LedgerAcquire.cpp | 8 +++++++- src/SHAMapSync.cpp | 6 +++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index c87d16ee3a..7a0892b523 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -8,6 +8,7 @@ #include "Application.h" #include "Log.h" #include "SHAMapSync.h" +#include "HashPrefixes.h" #define LA_DEBUG #define LEDGER_ACQUIRE_TIMEOUT 2 @@ -300,7 +301,12 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) return false; } mHaveBase = true; - theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), strCopy(data), mHash); + + Serializer s(data.size() + 4); + s.add32(sHP_Ledger); + s.addRaw(data); + theApp->getHashedObjectStore().store(LEDGER, mLedger->getLedgerSeq(), s.peekData(), mHash); + progress(); if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 6ebc64344b..b9a29210c3 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -234,7 +234,11 @@ bool SHAMap::addKnownNode(const SHAMapNode& node, const std::vectorgotNode(node, hash, rawNode, newNode->isLeaf()); + { + Serializer s; + newNode->addRaw(s, STN_ARF_PREFIXED); + filter->gotNode(node, hash, s.peekData(), newNode->isLeaf()); + } mTNByID[*newNode] = newNode; if (!newNode->isLeaf()) From 6a9b5ccd7d738ebd294c65400ceafa1f484f8611 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 23:19:29 -0700 Subject: [PATCH 04/14] Support STN_ARF_UNKNOWN for contexts where the format of a node is not known. Format is auto-detected. --- src/SHAMap.h | 7 ++++--- src/SHAMapNodes.cpp | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/SHAMap.h b/src/SHAMap.h index 2e1af1dcaf..64eec583a8 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -157,11 +157,12 @@ public: SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq); // copy node from older tree SHAMapTreeNode(const SHAMapNode& nodeID, SHAMapItem::pointer item, TNType type, uint32 seq); - // raw node functions - SHAMapTreeNode(const SHAMapNode& id, const std::vector& contents, uint32 seq, int format); - +#define STN_ARF_UNKNOWN 0 #define STN_ARF_PREFIXED 1 #define STN_ARF_WIRE 2 + + // raw node functions + SHAMapTreeNode(const SHAMapNode& id, const std::vector& contents, uint32 seq, int format); void addRaw(Serializer &, int format); virtual bool isPopulated() const { return true; } diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index dbdc8ee109..3a9c682414 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -192,6 +192,14 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& node, SHAMapItem::pointer item, SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector& rawNode, uint32 seq, int format) : SHAMapNode(id), mSeq(seq), mType(tnERROR), mFullBelow(false) { + if (format == STN_ARF_UNKNOWN) + { + if ((rawNode.size() < 4) || (rawNode[0] < 10)) + format = STN_ARF_WIRE; + else + format = STN_ARF_PREFIXED; + } + if (format == STN_ARF_WIRE) { Serializer s(rawNode); From ce14aa81914db875e4e047e98b81c50c0987cfb3 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 23:19:56 -0700 Subject: [PATCH 05/14] Make 'addRootNode' support both prefixed and wire formats. Fix getMissingNodes to retrieve from the node store in prefixed format. --- src/SHAMapSync.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index b9a29210c3..538ebfdb35 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -58,7 +58,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector nodeData; if (filter->haveNode(childID, childHash, nodeData)) { - d = boost::make_shared(childID, nodeData, mSeq, STN_ARF_WIRE); + d = boost::make_shared(childID, nodeData, mSeq, STN_ARF_PREFIXED); if (childHash != d->getNodeHash()) { Log(lsERROR) << "Wrong hash from cached object"; @@ -133,7 +133,7 @@ bool SHAMap::addRootNode(const std::vector& rootNode) return true; } - SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_WIRE); + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_UNKNOWN); if (!node) return false; #ifdef DEBUG @@ -165,7 +165,7 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& return true; } - SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_WIRE); + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_UNKNOWN); if (!node) return false; if (node->getNodeHash() != hash) From 0627ae22da7e8241a4d9903895ce017b840577d6 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 23:32:26 -0700 Subject: [PATCH 06/14] A better fix. Take the format as a parameter in addRootNode. --- src/LedgerAcquire.cpp | 4 ++-- src/SHAMap.h | 5 ++--- src/SHAMapNodes.cpp | 8 -------- src/SHAMapSync.cpp | 10 +++++----- 4 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 7a0892b523..8049ddfbd0 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -326,7 +326,7 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, { if (nodeIDit->isRoot()) { - if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait)) + if (!mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), *nodeDatait, STN_ARF_WIRE)) return false; } else if (!mLedger->peekTransactionMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) @@ -358,7 +358,7 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, { if (nodeIDit->isRoot()) { - if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait)) + if (!mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), *nodeDatait, STN_ARF_WIRE)) return false; } else if (!mLedger->peekAccountStateMap()->addKnownNode(*nodeIDit, *nodeDatait, &tFilter)) diff --git a/src/SHAMap.h b/src/SHAMap.h index 64eec583a8..f0083dd277 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -157,7 +157,6 @@ public: SHAMapTreeNode(const SHAMapTreeNode& node, uint32 seq); // copy node from older tree SHAMapTreeNode(const SHAMapNode& nodeID, SHAMapItem::pointer item, TNType type, uint32 seq); -#define STN_ARF_UNKNOWN 0 #define STN_ARF_PREFIXED 1 #define STN_ARF_WIRE 2 @@ -324,8 +323,8 @@ public: SHAMapSyncFilter* filter); bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, std::list >& rawNode, bool fatLeaves); - bool addRootNode(const uint256& hash, const std::vector& rootNode); - bool addRootNode(const std::vector& rootNode); + bool addRootNode(const uint256& hash, const std::vector& rootNode, int format); + bool addRootNode(const std::vector& rootNode, int format); bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode, SHAMapSyncFilter* filter); diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 3a9c682414..dbdc8ee109 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -192,14 +192,6 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& node, SHAMapItem::pointer item, SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector& rawNode, uint32 seq, int format) : SHAMapNode(id), mSeq(seq), mType(tnERROR), mFullBelow(false) { - if (format == STN_ARF_UNKNOWN) - { - if ((rawNode.size() < 4) || (rawNode[0] < 10)) - format = STN_ARF_WIRE; - else - format = STN_ARF_PREFIXED; - } - if (format == STN_ARF_WIRE) { Serializer s(rawNode); diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 538ebfdb35..548008a68b 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -122,7 +122,7 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI return true; } -bool SHAMap::addRootNode(const std::vector& rootNode) +bool SHAMap::addRootNode(const std::vector& rootNode, int format) { boost::recursive_mutex::scoped_lock sl(mLock); @@ -133,7 +133,7 @@ bool SHAMap::addRootNode(const std::vector& rootNode) return true; } - SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_UNKNOWN); + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, format); if (!node) return false; #ifdef DEBUG @@ -153,7 +153,7 @@ bool SHAMap::addRootNode(const std::vector& rootNode) return true; } -bool SHAMap::addRootNode(const uint256& hash, const std::vector& rootNode) +bool SHAMap::addRootNode(const uint256& hash, const std::vector& rootNode, int format) { boost::recursive_mutex::scoped_lock sl(mLock); @@ -165,7 +165,7 @@ bool SHAMap::addRootNode(const uint256& hash, const std::vector& return true; } - SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, STN_ARF_UNKNOWN); + SHAMapTreeNode::pointer node = boost::make_shared(SHAMapNode(), rootNode, 0, format); if (!node) return false; if (node->getNodeHash() != hash) @@ -447,7 +447,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) Log(lsFATAL) << "Didn't get root node " << gotNodes.size(); BOOST_FAIL("NodeSize"); } - if (!destination.addRootNode(*gotNodes.begin())) + if (!destination.addRootNode(*gotNodes.begin(), STN_ARF_WIRE)) { Log(lsFATAL) << "AddRootNode fails"; BOOST_FAIL("AddRootNode"); From c1d01348f88c6e8e1ba1bb6b7b6088e04b12b6d4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 29 Jun 2012 23:33:12 -0700 Subject: [PATCH 07/14] Add root node in wire format. --- src/LedgerConsensus.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 8da8844529..755b74c2c4 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -101,7 +101,7 @@ bool TransactionAcquire::takeNodes(const std::list& nodeIDs, Log(lsWARNING) << "Got root TXS node, already have it"; return false; } - if (!mMap->addRootNode(getHash(), *nodeDatait)) + if (!mMap->addRootNode(getHash(), *nodeDatait, STN_ARF_WIRE)) return false; else mHaveRoot = true; } From 64a869d6e81ce8d4229b0b0ec5a6e3c3c0f0b65f Mon Sep 17 00:00:00 2001 From: jed Date: Sat, 30 Jun 2012 05:58:45 -0700 Subject: [PATCH 08/14] less chatty --- src/ConnectionPool.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/ConnectionPool.cpp b/src/ConnectionPool.cpp index 4d666ef221..6497ede5f6 100644 --- a/src/ConnectionPool.cpp +++ b/src/ConnectionPool.cpp @@ -544,7 +544,7 @@ void ConnectionPool::peerVerified(Peer::pointer peer) std::string strIpPort = str(boost::format("%s %d") % strIp % iPort); - Log(lsINFO) << str(boost::format("Pool: Scan: connected: %s %s %s (scanned)") % ADDRESS_SHARED(peer) % strIp % iPort); + //Log(lsINFO) << str(boost::format("Pool: Scan: connected: %s %s %s (scanned)") % ADDRESS_SHARED(peer) % strIp % iPort); if (peer->getNodePublic() == theApp->getWallet().getNodePublic()) { @@ -628,7 +628,7 @@ void ConnectionPool::scanRefresh() if (tpNow.is_not_a_date_time()) { - Log(lsINFO) << "Pool: Scan: stop."; + //Log(lsINFO) << "Pool: Scan: stop."; (void) mScanTimer.cancel(); } @@ -643,8 +643,8 @@ void ConnectionPool::scanRefresh() tpNext = tpNow + boost::posix_time::seconds(iInterval); - Log(lsINFO) << str(boost::format("Pool: Scan: Now: %s %s (next %s, delay=%d)") - % mScanIp % mScanPort % tpNext % (tpNext-tpNow).total_seconds()); + //Log(lsINFO) << str(boost::format("Pool: Scan: Now: %s %s (next %s, delay=%d)") + // % mScanIp % mScanPort % tpNext % (tpNext-tpNow).total_seconds()); iInterval *= 2; @@ -668,8 +668,8 @@ void ConnectionPool::scanRefresh() } else { - Log(lsINFO) << str(boost::format("Pool: Scan: Next: %s (next %s, delay=%d)") - % strIpPort % tpNext % (tpNext-tpNow).total_seconds()); + //Log(lsINFO) << str(boost::format("Pool: Scan: Next: %s (next %s, delay=%d)") + // % strIpPort % tpNext % (tpNext-tpNow).total_seconds()); mScanTimer.expires_at(tpNext); mScanTimer.async_wait(boost::bind(&ConnectionPool::scanHandler, this, _1)); From 8f7c984ef764c0379143a45bbb421c55e7a199a8 Mon Sep 17 00:00:00 2001 From: jed Date: Sat, 30 Jun 2012 06:52:05 -0700 Subject: [PATCH 09/14] don't validate if ledger # is too low --- src/Config.cpp | 4 ++++ src/Config.h | 3 +++ src/NetworkOPs.cpp | 13 +++++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Config.cpp b/src/Config.cpp index 6e4b67ce87..0ca3ec46cd 100644 --- a/src/Config.cpp +++ b/src/Config.cpp @@ -121,6 +121,7 @@ void Config::setup(const std::string& strConf) // a new ledger every minute LEDGER_SECONDS = 60; + LEDGER_CREATOR = false; RPC_USER = "admin"; RPC_PASSWORD = "pass"; @@ -201,6 +202,9 @@ void Config::load() if (sectionSingleB(secConfig, SECTION_RPC_PORT, strTemp)) RPC_PORT = boost::lexical_cast(strTemp); + if (sectionSingleB(secConfig, "ledger_creator" , strTemp)) + LEDGER_CREATOR = boost::lexical_cast(strTemp); + if (sectionSingleB(secConfig, SECTION_RPC_ALLOW_REMOTE, strTemp)) RPC_ALLOW_REMOTE = boost::lexical_cast(strTemp); diff --git a/src/Config.h b/src/Config.h index 6a88d13cfe..7c900523f5 100644 --- a/src/Config.h +++ b/src/Config.h @@ -54,12 +54,15 @@ public: std::vector VALIDATORS; // Validators from newcoind.cfg. std::vector IPS; // Peer IPs from newcoind.cfg. + + // Network parameters int NETWORK_START_TIME; // The Unix time we start ledger 0. int TRANSACTION_FEE_BASE; int LEDGER_SECONDS; int LEDGER_PROPOSAL_DELAY_SECONDS; int LEDGER_AVALANCHE_SECONDS; + bool LEDGER_CREATOR; // should be false unless we are starting a new ledger // Note: The following parameters do not relate to the UNL or trust at all int NETWORK_QUORUM; // Minimum number of nodes to consider the network present diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index c29da13f69..4d3485c479 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -369,10 +369,15 @@ bool NetworkOPs::checkLastClosedLedger(const std::vector& peerLis } Ledger::pointer ourClosed = mLedgerMaster->getClosedLedger(); - uint256 closedLedger = ourClosed->getHash(); - ValidationCount& ourVC = ledgers[closedLedger]; - ++ourVC.nodesUsing; - ourVC.highNode = theApp->getWallet().getNodePublic(); + uint256 closedLedger=0; + if(theConfig.LEDGER_CREATOR || ourClosed->getLedgerSeq() > 100) + { + closedLedger = ourClosed->getHash(); + ValidationCount& ourVC = ledgers[closedLedger]; + ++ourVC.nodesUsing; + ourVC.highNode = theApp->getWallet().getNodePublic(); + } + for (std::vector::const_iterator it = peerList.begin(), end = peerList.end(); it != end; ++it) { From ae1e2e071e81d37f621a4d127f36997b56ae4121 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 30 Jun 2012 18:52:52 -0700 Subject: [PATCH 10/14] Fix a crash bug Jed reported. --- src/SHAMapNodes.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index dbdc8ee109..af990f6b40 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -255,15 +255,16 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector(Serializer::getSHA512Half(rawNode), s.peekData()); + mItem = boost::make_shared(s.getSHA512Half(), s.peekData()); mType = tnTRANSACTION; } if (prefix == sHP_LeafNode) { uint256 u; s.get256(u, s.getLength() - 32); - s.chop(256 / 8); - if (u.isZero()) throw std::runtime_error("invalid PLN node"); + s.chop(32); + if (u.isZero()) + throw std::runtime_error("invalid PLN node"); mItem = boost::make_shared(u, s.peekData()); mType = tnACCOUNT_STATE; } From 30d76c87a39253128596270ced1ba570d3f7d36e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 30 Jun 2012 18:53:11 -0700 Subject: [PATCH 11/14] Don't crash if we're send a malicious map node from a peer. --- src/SHAMap.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index cb48e8df5e..cc27ed70b6 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -630,11 +630,19 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui throw SHAMapMissingNode(id, hash); assert(Serializer::getSHA512Half(obj->getData()) == hash); - SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq, STN_ARF_PREFIXED); + try + { + SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq, STN_ARF_PREFIXED); #ifdef DEBUG - assert((ret->getNodeHash() == hash) && (id == *ret)); + assert((ret->getNodeHash() == hash) && (id == *ret)); #endif - return ret; + return ret; + } + catch (...) + { + Log(lsWARNING) << "fetchNodeExternal gets an invalid node: " << hash.GetHex(); + throw SHAMapMissingNode(id, hash); + } } void SHAMap::armDirty() From c90f834f7f2eda481352972600728284c7020581 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 30 Jun 2012 18:59:38 -0700 Subject: [PATCH 12/14] Duh! This is the right fix for the bug Jed reported. --- src/SHAMapNodes.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index af990f6b40..32d9b65d80 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -247,7 +247,10 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector(s.getSHA512Half(), s.peekData()); mType = tnTRANSACTION; } - if (prefix == sHP_LeafNode) + else if (prefix == sHP_LeafNode) { uint256 u; s.get256(u, s.getLength() - 32); s.chop(32); if (u.isZero()) + { + Log(lsINFO) << "invalid PLN node"; throw std::runtime_error("invalid PLN node"); + } mItem = boost::make_shared(u, s.peekData()); mType = tnACCOUNT_STATE; } - if (prefix == sHP_InnerNode) + else if (prefix == sHP_InnerNode) { if (rawNode.size() != (512 + 4)) throw std::runtime_error("invalid PIN node"); @@ -277,7 +283,10 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector Date: Sat, 30 Jun 2012 19:07:08 -0700 Subject: [PATCH 13/14] Keep and report some extra peer information. --- src/Peer.cpp | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/Peer.cpp b/src/Peer.cpp index 35da711dad..2ce12c4e1b 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -870,7 +870,15 @@ void Peer::recvStatus(newcoin::TMStatusChange& packet) Log(lsTRACE) << "Received status change from peer " << getIP(); if (!packet.has_networktime()) packet.set_networktime(theApp->getOPs().getNetworkTimeNC()); - mLastStatus = packet; + + if (!mLastStatus.has_newstatus() || packet.has_newstatus()) + mLastStatus = packet; + else + { // preserve old status + newcoin::NodeStatus status = mLastStatus.newstatus(); + mLastStatus = packet; + packet.set_newstatus(status); + } if (packet.newevent() == newcoin::neLOST_SYNC) { @@ -1165,6 +1173,23 @@ Json::Value Peer::getJson() (mHello.protoversion() != MAKE_VERSION_INT(PROTO_VERSION_MAJOR, PROTO_VERSION_MINOR))) ret["protocol"] = boost::lexical_cast(GET_VERSION_MAJOR(mHello.protoversion())) + "." + boost::lexical_cast(GET_VERSION_MINOR(mHello.protoversion())); + + if (!!mClosedLedgerHash) + ret["ledger"] = mClosedLedgerHash.GetHex(); + + if (mLastStatus.has_newstatus()) + { + switch (mLastStatus.newstatus()) + { + case newcoin::nsCONNECTING: ret["status"] = "connecting"; break; + case newcoin::nsCONNECTED: ret["status"] = "connected"; break; + case newcoin::nsMONITORING: ret["status"] = "monitoring"; break; + case newcoin::nsVALIDATING: ret["status"] = "validating"; break; + case newcoin::nsSHUTTING: ret["status"] = "shutting"; break; + default: Log(lsWARNING) << "Peer has unknown status: " << mLastStatus.newstatus(); + } + } + /* if (!mIpPort.first.empty()) { From c7eb0269c2c0b28a1f465f9d01a66aca409ddd33 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 30 Jun 2012 22:17:53 -0700 Subject: [PATCH 14/14] For now, detect clocks out of sync and disallow connection. Improve speed of ledger synchronization by compressing all three base queries into one. --- src/LedgerAcquire.cpp | 61 +++++++++++++++++++++++++++++++++++-------- src/LedgerAcquire.h | 12 ++++----- src/NetworkOPs.cpp | 5 ++-- src/Peer.cpp | 34 +++++++++++++++++++++++- src/SHAMap.h | 1 + src/SHAMapSync.cpp | 7 +++++ 6 files changed, 100 insertions(+), 20 deletions(-) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 8049ddfbd0..ffd82a7f83 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -138,6 +138,7 @@ void LedgerAcquire::trigger(Peer::pointer peer) newcoin::TMGetLedger tmGL; tmGL.set_ledgerhash(mHash.begin(), mHash.size()); tmGL.set_itype(newcoin::liBASE); + *(tmGL.add_nodeids()) = SHAMapNode().getRawString(); if (peer) { sendRequest(tmGL, peer); @@ -285,7 +286,7 @@ void PeerSet::sendRequest(const newcoin::TMGetLedger& tmGL) } } -bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) +bool LedgerAcquire::takeBase(const std::string& data) { // Return value: true=normal, false=bad data #ifdef LA_DEBUG Log(lsTRACE) << "got base acquiring ledger " << mHash.GetHex(); @@ -311,12 +312,11 @@ bool LedgerAcquire::takeBase(const std::string& data, Peer::pointer peer) if (!mLedger->getTransHash()) mHaveTransactions = true; if (!mLedger->getAccountHash()) mHaveState = true; mLedger->setAcquiring(); - trigger(peer); return true; } bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, - const std::list< std::vector >& data, Peer::pointer peer) + const std::list< std::vector >& data) { if (!mHaveBase) return false; std::list::const_iterator nodeIDit = nodeIDs.begin(); @@ -339,13 +339,12 @@ bool LedgerAcquire::takeTxNode(const std::list& nodeIDs, mHaveTransactions = true; if (mHaveState) mComplete = true; } - trigger(peer); progress(); return true; } bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, - const std::list< std::vector >& data, Peer::pointer peer) + const std::list< std::vector >& data) { #ifdef LA_DEBUG Log(lsTRACE) << "got ASdata acquiring ledger " << mHash.GetHex(); @@ -371,11 +370,22 @@ bool LedgerAcquire::takeAsNode(const std::list& nodeIDs, mHaveState = true; if (mHaveTransactions) mComplete = true; } - trigger(peer); progress(); return true; } +bool LedgerAcquire::takeAsRootNode(const std::vector& data) +{ + if (!mHaveBase) return false; + return mLedger->peekAccountStateMap()->addRootNode(mLedger->getAccountHash(), data, STN_ARF_WIRE); +} + +bool LedgerAcquire::takeTxRootNode(const std::vector& data) +{ + if (!mHaveBase) return false; + return mLedger->peekTransactionMap()->addRootNode(mLedger->getTransHash(), data, STN_ARF_WIRE); +} + LedgerAcquire::pointer LedgerAcquireMaster::findCreate(const uint256& hash) { boost::mutex::scoped_lock sl(mLock); @@ -428,11 +438,34 @@ bool LedgerAcquireMaster::gotLedgerData(newcoin::TMLedgerData& packet, Peer::poi if (packet.type() == newcoin::liBASE) { - if (packet.nodes_size() != 1) return false; + if (packet.nodes_size() < 1) + return false; const newcoin::TMLedgerNode& node = packet.nodes(0); - return ledger->takeBase(node.nodedata(), peer); + if (!ledger->takeBase(node.nodedata())) + return false; + if (packet.nodes_size() == 1) + { + ledger->trigger(peer); + return true; + } + Log(lsDEBUG) << "liBASE includes ASbase"; + if (!ledger->takeAsRootNode(strCopy(packet.nodes(1).nodedata()))) + { + Log(lsWARNING) << "Included ASbase invalid"; + } + if (packet.nodes().size() == 2) + { + ledger->trigger(peer); + return true; + } + Log(lsDEBUG) << "liBASE includes TXbase"; + if (!ledger->takeTxRootNode(strCopy(packet.nodes(2).nodedata()))) + Log(lsWARNING) << "Invcluded TXbase invalid"; + ledger->trigger(peer); + return true; } - else if ((packet.type() == newcoin::liTX_NODE) || (packet.type() == newcoin::liAS_NODE)) + + if ((packet.type() == newcoin::liTX_NODE) || (packet.type() == newcoin::liAS_NODE)) { std::list nodeIDs; std::list< std::vector > nodeData; @@ -446,8 +479,14 @@ bool LedgerAcquireMaster::gotLedgerData(newcoin::TMLedgerData& packet, Peer::poi nodeIDs.push_back(SHAMapNode(node.nodeid().data(), node.nodeid().size())); nodeData.push_back(std::vector(node.nodedata().begin(), node.nodedata().end())); } - if (packet.type() == newcoin::liTX_NODE) return ledger->takeTxNode(nodeIDs, nodeData, peer); - else return ledger->takeAsNode(nodeIDs, nodeData, peer); + bool ret; + if (packet.type() == newcoin::liTX_NODE) + ret = ledger->takeTxNode(nodeIDs, nodeData); + else + ret = ledger->takeAsNode(nodeIDs, nodeData); + if (ret) + ledger->trigger(peer); + return ret; } return false; diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index cc499fdb3a..ba96139372 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -73,7 +73,6 @@ protected: void onTimer() { trigger(Peer::pointer()); } void newPeer(Peer::pointer peer) { trigger(peer); } - void trigger(Peer::pointer); boost::weak_ptr pmDowncast(); @@ -87,11 +86,12 @@ public: void addOnComplete(boost::function); - bool takeBase(const std::string& data, Peer::pointer); - bool takeTxNode(const std::list& IDs, const std::list >& data, - Peer::pointer); - bool takeAsNode(const std::list& IDs, const std::list >& data, - Peer::pointer); + bool takeBase(const std::string& data); + bool takeTxNode(const std::list& IDs, const std::list >& data); + bool takeTxRootNode(const std::vector& data); + bool takeAsNode(const std::list& IDs, const std::list >& data); + bool takeAsRootNode(const std::vector& data); + void trigger(Peer::pointer); }; class LedgerAcquireMaster diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 4d3485c479..3800eadd13 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -388,7 +388,7 @@ bool NetworkOPs::checkLastClosedLedger(const std::vector& peerLis else if ((*it)->isConnected()) { uint256 peerLedger = (*it)->getClosedLedgerHash(); - if (!!peerLedger) + if (peerLedger.isNonZero()) { ValidationCount& vc = ledgers[peerLedger]; if ((vc.nodesUsing == 0) || ((*it)->getNodePublic() > vc.highNode)) @@ -513,7 +513,8 @@ int NetworkOPs::beginConsensus(const uint256& networkClosed, Ledger::pointer clo assert(closingLedger->getParentHash() == mLedgerMaster->getClosedLedger()->getHash()); // Create a consensus object to get consensus on this ledger - if (!!mConsensus) mConsensus->abort(); + if (!!mConsensus) + mConsensus->abort(); prevLedger->setImmutable(); mConsensus = boost::make_shared( networkClosed, diff --git a/src/Peer.cpp b/src/Peer.cpp index 2ce12c4e1b..e15a654dd5 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -571,7 +571,14 @@ void Peer::recvHello(newcoin::TMHello& packet) // Cancel verification timeout. (void) mVerifyTimer.cancel(); - if (packet.protoversionmin() < MAKE_VERSION_INT(MIN_PROTO_MAJOR, MIN_PROTO_MINOR)) + uint64 minTime = theApp->getOPs().getNetworkTimeNC() - 4; + uint64 maxTime = minTime + 8; + + if (packet.has_nettime() && ((packet.nettime() < minTime) || (packet.nettime() > maxTime))) + { + Log(lsINFO) << "Recv(Hello): Disconnect: Clocks are too far off"; + } + else if (packet.protoversionmin() < MAKE_VERSION_INT(MIN_PROTO_MAJOR, MIN_PROTO_MINOR)) { Log(lsINFO) << "Recv(Hello): Server requires protocol version " << GET_VERSION_MAJOR(packet.protoversion()) << "." << GET_VERSION_MINOR(packet.protoversion()) @@ -987,6 +994,31 @@ void Peer::recvGetLedger(newcoin::TMGetLedger& packet) Serializer nData(128); ledger->addRaw(nData); reply.add_nodes()->set_nodedata(nData.getDataPtr(), nData.getLength()); + + if (packet.nodeids().size() != 0) + { + Log(lsINFO) << "Ledger root w/map roots request"; + SHAMap::pointer map = ledger->peekAccountStateMap(); + if (map) + { + Serializer rootNode(768); + if (map->getRootNode(rootNode, STN_ARF_WIRE)) + { + reply.add_nodes()->set_nodedata(rootNode.getDataPtr(), rootNode.getLength()); + if (ledger->getTransHash().isNonZero()) + { + map = ledger->peekTransactionMap(); + if (map) + { + rootNode.resize(0); + if (map->getRootNode(rootNode, STN_ARF_WIRE)) + reply.add_nodes()->set_nodedata(rootNode.getDataPtr(), rootNode.getLength()); + } + } + } + } + } + PackedMessage::pointer oPacket = boost::make_shared(reply, newcoin::mtLEDGER_DATA); sendPacket(oPacket); return; diff --git a/src/SHAMap.h b/src/SHAMap.h index f0083dd277..08685a6b39 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -323,6 +323,7 @@ public: SHAMapSyncFilter* filter); bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, std::list >& rawNode, bool fatLeaves); + bool getRootNode(Serializer& s, int format); bool addRootNode(const uint256& hash, const std::vector& rootNode, int format); bool addRootNode(const std::vector& rootNode, int format); bool addKnownNode(const SHAMapNode& nodeID, const std::vector& rawNode, diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index 548008a68b..763dd15147 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -122,6 +122,13 @@ bool SHAMap::getNodeFat(const SHAMapNode& wanted, std::vector& nodeI return true; } +bool SHAMap::getRootNode(Serializer& s, int format) +{ + boost::recursive_mutex::scoped_lock sl(mLock); + root->addRaw(s, format); + return true; +} + bool SHAMap::addRootNode(const std::vector& rootNode, int format) { boost::recursive_mutex::scoped_lock sl(mLock);