diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index 715a84a33..97a766cfe 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -58,7 +58,7 @@ Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint25 try { if (mTransHash.isNonZero()) - mTransactionMap->fetchRoot(mTransHash); + mTransactionMap->fetchRoot(mTransHash, NULL); } catch (...) { @@ -68,7 +68,7 @@ Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint25 try { if (mAccountHash.isNonZero()) - mAccountStateMap->fetchRoot(mAccountHash); + mAccountStateMap->fetchRoot(mAccountHash, NULL); } catch (...) { @@ -1625,7 +1625,7 @@ uint64 Ledger::scaleFeeLoad(uint64 fee, bool bAdmin) return theApp->getFeeTrack().scaleFeeLoad(fee, mBaseFee, mReferenceFeeUnits, bAdmin); } -std::vector Ledger::getNeededTransactionHashes(int max) +std::vector Ledger::getNeededTransactionHashes(int max, SHAMapSyncFilter* filter) { std::vector ret; if (mTransHash.isNonZero()) @@ -1633,12 +1633,12 @@ std::vector Ledger::getNeededTransactionHashes(int max) if (mTransactionMap->getHash().isZero()) ret.push_back(mTransHash); else - ret = mTransactionMap->getNeededHashes(max); + ret = mTransactionMap->getNeededHashes(max, filter); } return ret; } -std::vector Ledger::getNeededAccountStateHashes(int max) +std::vector Ledger::getNeededAccountStateHashes(int max, SHAMapSyncFilter* filter) { std::vector ret; if (mAccountHash.isNonZero()) @@ -1646,7 +1646,7 @@ std::vector Ledger::getNeededAccountStateHashes(int max) if (mAccountStateMap->getHash().isZero()) ret.push_back(mAccountHash); else - ret = mAccountStateMap->getNeededHashes(max); + ret = mAccountStateMap->getNeededHashes(max, filter); } return ret; } diff --git a/src/cpp/ripple/Ledger.h b/src/cpp/ripple/Ledger.h index 3fb0a898f..6b3399968 100644 --- a/src/cpp/ripple/Ledger.h +++ b/src/cpp/ripple/Ledger.h @@ -222,8 +222,8 @@ public: static uint256 getLedgerFeatureIndex(); static uint256 getLedgerFeeIndex(); - std::vector getNeededTransactionHashes(int max); - std::vector getNeededAccountStateHashes(int max); + std::vector getNeededTransactionHashes(int max, SHAMapSyncFilter* filter); + std::vector getNeededAccountStateHashes(int max, SHAMapSyncFilter* filter); // index calculation functions static uint256 getAccountRootIndex(const uint160& uAccountID); diff --git a/src/cpp/ripple/LedgerAcquire.cpp b/src/cpp/ripple/LedgerAcquire.cpp index 3eedaf7e1..e2f5be8f5 100644 --- a/src/cpp/ripple/LedgerAcquire.cpp +++ b/src/cpp/ripple/LedgerAcquire.cpp @@ -114,9 +114,18 @@ bool LedgerAcquire::tryLocal() // Nothing we can do without the ledger base HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(mHash); if (!node) - return false; + { + std::vector data; + if (!theApp->getOPs().getFetchPack(mHash, data)) + return false; + mLedger = boost::make_shared(data, true); + theApp->getHashedObjectStore().store(hotLEDGER, mLedger->getLedgerSeq(), data, mHash); + } + else + { + mLedger = boost::make_shared(strCopy(node->getData()), true); + } - mLedger = boost::make_shared(strCopy(node->getData()), true); if (mLedger->getHash() != mHash) { // We know for a fact the ledger can never be acquired cLog(lsWARNING) << mHash << " cannot be a ledger"; @@ -134,9 +143,10 @@ bool LedgerAcquire::tryLocal() { try { - mLedger->peekTransactionMap()->fetchRoot(mLedger->getTransHash()); + TransactionStateSF filter(mLedger->getLedgerSeq()); + mLedger->peekTransactionMap()->fetchRoot(mLedger->getTransHash(), &filter); cLog(lsDEBUG) << "Got root txn map locally"; - std::vector h = mLedger->getNeededTransactionHashes(1); + std::vector h = mLedger->getNeededTransactionHashes(1, &filter); if (h.empty()) { cLog(lsDEBUG) << "Had full txn map locally"; @@ -157,9 +167,10 @@ bool LedgerAcquire::tryLocal() { try { - mLedger->peekAccountStateMap()->fetchRoot(mLedger->getAccountHash()); + AccountStateSF filter(mLedger->getLedgerSeq()); + mLedger->peekAccountStateMap()->fetchRoot(mLedger->getAccountHash(), &filter); cLog(lsDEBUG) << "Got root AS map locally"; - std::vector h = mLedger->getNeededAccountStateHashes(1); + std::vector h = mLedger->getNeededAccountStateHashes(1, &filter); if (h.empty()) { cLog(lsDEBUG) << "Had full AS map locally"; @@ -809,13 +820,15 @@ std::vector LedgerAcquire::getNeededHashes() } if (!mHaveState) { - std::vector v = mLedger->getNeededAccountStateHashes(4); + AccountStateSF filter(mLedger->getLedgerSeq()); + std::vector v = mLedger->getNeededAccountStateHashes(4, &filter); BOOST_FOREACH(const uint256& h, v) ret.push_back(std::make_pair(ripple::TMGetObjectByHash::otSTATE_NODE, h)); } if (!mHaveTransactions) { - std::vector v = mLedger->getNeededAccountStateHashes(4); + TransactionStateSF filter(mLedger->getLedgerSeq()); + std::vector v = mLedger->getNeededAccountStateHashes(4, &filter); BOOST_FOREACH(const uint256& h, v) ret.push_back(std::make_pair(ripple::TMGetObjectByHash::otTRANSACTION_NODE, h)); } @@ -839,7 +852,7 @@ Json::Value LedgerAcquire::getJson(int) if (mHaveBase && !mHaveState) { Json::Value hv(Json::arrayValue); - std::vector v = mLedger->peekAccountStateMap()->getNeededHashes(16); + std::vector v = mLedger->peekAccountStateMap()->getNeededHashes(16, NULL); BOOST_FOREACH(const uint256& h, v) hv.append(h.GetHex()); ret["needed_state_hashes"] = hv; @@ -847,7 +860,7 @@ Json::Value LedgerAcquire::getJson(int) if (mHaveBase && !mHaveTransactions) { Json::Value hv(Json::arrayValue); - std::vector v = mLedger->peekTransactionMap()->getNeededHashes(16); + std::vector v = mLedger->peekTransactionMap()->getNeededHashes(16, NULL); BOOST_FOREACH(const uint256& h, v) hv.append(h.GetHex()); ret["needed_transaction_hashes"] = hv; diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index 4878f15eb..af56d2824 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -474,6 +474,12 @@ void LedgerConsensus::statusChange(ripple::NodeEvent event, Ledger& ledger) s.set_ledgerhashprevious(hash.begin(), hash.size()); hash = ledger.getHash(); s.set_ledgerhash(hash.begin(), hash.size()); + + uint32 uMin, uMax; + theApp->getOPs().getValidatedRange(uMin, uMax); + s.set_firstseq(uMin); + s.set_lastseq(uMax); + PackedMessage::pointer packet = boost::make_shared(s, ripple::mtSTATUS_CHANGE); theApp->getConnectionPool().relayMessage(NULL, packet); cLog(lsTRACE) << "send status change to peer"; diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index f7169777f..b20ba7c5d 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -251,6 +251,41 @@ bool LedgerMaster::acquireMissingLedger(Ledger::ref origLedger, const uint256& l theApp->getIOService().post(boost::bind(&LedgerMaster::missingAcquireComplete, this, mMissingLedger)); } + if (theApp->getOPs().getFetchSize() < 256) + { // refill our fetch pack + Ledger::pointer nextLedger = mLedgerHistory.getLedgerBySeq(ledgerSeq + 1); + if (nextLedger) + { + ripple::TMGetObjectByHash tmBH; + tmBH.set_type(ripple::TMGetObjectByHash::otFETCH_PACK); + tmBH.set_query(true); + tmBH.set_seq(ledgerSeq); + tmBH.set_ledgerhash(ledgerHash.begin(), 32); + std::vector peerList = theApp->getConnectionPool().getPeerVector(); + + Peer::pointer target; + int count = 0; + + BOOST_FOREACH(const Peer::pointer& peer, peerList) + { + if (peer->hasRange(ledgerSeq, ledgerSeq + 1)) + { + if (count++ == 0) + target = peer; + else if ((rand() % count) == 0) + target = peer; + } + } + if (target) + { + PackedMessage::pointer packet = boost::make_shared(tmBH, ripple::mtGET_OBJECTS); + target->sendPacket(packet, false); + } + else + cLog(lsINFO) << "No peer for fetch pack"; + } + } + int fetchMax = theConfig.getSize(siLedgerFetch); int timeoutCount; int fetchCount = theApp->getMasterLedgerAcquire().getFetchCount(timeoutCount); diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index bb67e40c4..922f9b07f 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -2058,7 +2058,21 @@ void NetworkOPs::addFetchPack(const uint256& hash, boost::shared_ptr< std::vecto bool NetworkOPs::getFetchPack(const uint256& hash, std::vector& data) { - return mFetchPack.retrieve(hash, data); + bool ret = mFetchPack.retrieve(hash, data); + if (!ret) + return false; + if (hash != Serializer::getSHA512Half(data)) + { + cLog(lsWARNING) << "Bad entry in fetch pack"; + return false; + } + cLog(lsTRACE) << hash << " found in fetch pack"; + return true; +} + +int NetworkOPs::getFetchSize() +{ + return mFetchPack.getCacheSize(); } // vim:ts=4 diff --git a/src/cpp/ripple/NetworkOPs.h b/src/cpp/ripple/NetworkOPs.h index 1fcd44cf9..1d80bdcaf 100644 --- a/src/cpp/ripple/NetworkOPs.h +++ b/src/cpp/ripple/NetworkOPs.h @@ -263,6 +263,7 @@ public: Ledger::pointer wantLedger, Ledger::pointer haveLedger); void addFetchPack(const uint256& hash, boost::shared_ptr< std::vector >& data); bool getFetchPack(const uint256& hash, std::vector& data); + int getFetchSize(); void sweepFetchPack(); // network state machine diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index dd9c0048f..ad49d9af9 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -36,6 +36,8 @@ Peer::Peer(boost::asio::io_service& io_service, boost::asio::ssl::context& ctx, mPeerId(peerID), mPrivate(false), mLoad(""), + mMinLedger(0), + mMaxLedger(0), mSocketSsl(io_service, ctx), mActivityTimer(io_service), mIOStrand(io_service) @@ -1381,6 +1383,11 @@ void Peer::recvStatus(ripple::TMStatusChange& packet) addLedger(mPreviousLedgerHash); } else mPreviousLedgerHash.zero(); + + if (packet.has_firstseq()) + mMinLedger = packet.firstseq(); + if (packet.has_lastseq()) + mMaxLedger = packet.lastseq(); } void Peer::recvGetLedger(ripple::TMGetLedger& packet) @@ -1877,6 +1884,11 @@ void Peer::doFetchPack(const boost::shared_ptr& packe } } +bool Peer::hasProto(int version) +{ + return mHello.has_protoversion() && (mHello.protoversion() >= version); +} + Json::Value Peer::getJson() { Json::Value ret(Json::objectValue); diff --git a/src/cpp/ripple/Peer.h b/src/cpp/ripple/Peer.h index bace95d35..d0c48b2cd 100644 --- a/src/cpp/ripple/Peer.h +++ b/src/cpp/ripple/Peer.h @@ -47,6 +47,7 @@ private: uint64 mPeerId; bool mPrivate; // Keep peer IP private. LoadSource mLoad; + uint32 mMinLedger, mMaxLedger; uint256 mClosedLedgerHash; uint256 mPreviousLedgerHash; @@ -162,6 +163,8 @@ public: const RippleAddress& getNodePublic() const { return mNodePublic; } void cycleStatus() { mPreviousLedgerHash = mClosedLedgerHash; mClosedLedgerHash.zero(); } + bool hasProto(int version); + bool hasRange(uint32 uMin, uint32 uMax) { return (uMin >= mMinLedger) && (uMax <= mMaxLedger); } }; #endif diff --git a/src/cpp/ripple/SHAMap.cpp b/src/cpp/ripple/SHAMap.cpp index 7a2b91ed9..e18bc225c 100644 --- a/src/cpp/ripple/SHAMap.cpp +++ b/src/cpp/ripple/SHAMap.cpp @@ -731,7 +731,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui } } -void SHAMap::fetchRoot(const uint256& hash) +void SHAMap::fetchRoot(const uint256& hash, SHAMapSyncFilter* filter) { if (sLog(lsTRACE)) { @@ -742,7 +742,20 @@ void SHAMap::fetchRoot(const uint256& hash) else cLog(lsTRACE) << "Fetch root SHAMap node " << hash; } - root = fetchNodeExternal(SHAMapNode(), hash); + try + { + root = fetchNodeExternal(SHAMapNode(), hash); + } + catch (SHAMapMissingNode& mn) + { + std::vector nodeData; + if (!filter || filter->haveNode(SHAMapNode(), hash, nodeData)) + throw; + root = boost::make_shared(SHAMapNode(), nodeData, + mSeq - 1, snfPREFIX, hash, true); + mTNByID[*root] = root; + filter->gotNode(true, SHAMapNode(), hash, nodeData, root->getType()); + } assert(root->getNodeHash() == hash); } diff --git a/src/cpp/ripple/SHAMap.h b/src/cpp/ripple/SHAMap.h index f228f73bb..adf0a76cb 100644 --- a/src/cpp/ripple/SHAMap.h +++ b/src/cpp/ripple/SHAMap.h @@ -21,6 +21,7 @@ DEFINE_INSTANCE(SHAMapItem); DEFINE_INSTANCE(SHAMapTreeNode); class SHAMap; +class SHAMapSyncFilter; // A tree-like map of SHA256 hashes // The trees are designed for rapid synchronization and compression of differences @@ -253,7 +254,7 @@ public: SHAMapSyncFilter() { ; } virtual ~SHAMapSyncFilter() { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + virtual void gotNode(bool fromFilter, const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, SHAMapTreeNode::TNType type) { ; } @@ -397,7 +398,7 @@ public: ScopedLock Lock() const { return ScopedLock(mLock); } bool hasNode(const SHAMapNode& id); - void fetchRoot(const uint256& hash); + void fetchRoot(const uint256& hash, SHAMapSyncFilter* filter); // normal hash access functions bool hasItem(const uint256& id); @@ -431,7 +432,7 @@ public: bool getNodeFat(const SHAMapNode& node, std::vector& nodeIDs, std::list >& rawNode, bool fatRoot, bool fatLeaves); bool getRootNode(Serializer& s, SHANodeFormat format); - std::vector getNeededHashes(int max); + std::vector getNeededHashes(int max, SHAMapSyncFilter* filter); SMAddNode addRootNode(const uint256& hash, const std::vector& rootNode, SHANodeFormat format, SHAMapSyncFilter* filter); SMAddNode addRootNode(const std::vector& rootNode, SHANodeFormat format, diff --git a/src/cpp/ripple/SHAMapSync.cpp b/src/cpp/ripple/SHAMapSync.cpp index 7156d9fef..5d1c372b1 100644 --- a/src/cpp/ripple/SHAMapSync.cpp +++ b/src/cpp/ripple/SHAMapSync.cpp @@ -69,10 +69,12 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector= 1); SHAMapTreeNode::pointer ptr = - boost::make_shared(childID, nodeData, mSeq - 1, snfPREFIX, childHash, true); + boost::make_shared(childID, nodeData, mSeq - 1, + snfPREFIX, childHash, true); cLog(lsTRACE) << "Got sync node from cache: " << *ptr; mTNByID[*ptr] = ptr; d = ptr.get(); + filter->gotNode(true, childID, childHash, nodeData, ptr->getType()); } } } @@ -106,7 +108,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector SHAMap::getNeededHashes(int max) +std::vector SHAMap::getNeededHashes(int max, SHAMapSyncFilter* filter) { std::vector ret; boost::recursive_mutex::scoped_lock sl(mLock); @@ -135,20 +137,38 @@ std::vector SHAMap::getNeededHashes(int max) if (!node->isEmptyBranch(branch)) { const uint256& childHash = node->getChildHash(branch); + SHAMapNode childID = node->getChildNodeID(branch); if (!fullBelowCache.isPresent(childHash)) { + SHAMapTreeNode* d = NULL; try { - SHAMapTreeNode* d = getNodePointer(node->getChildNodeID(branch), childHash); + d = getNodePointer(node->getChildNodeID(branch), childHash); assert(d); + } + catch (SHAMapMissingNode&) + { // node is not in the map + std::vector nodeData; + if (filter && filter->haveNode(childID, childHash, nodeData)) + { + SHAMapTreeNode::pointer ptr = + boost::make_shared(childID, nodeData, mSeq -1, + snfPREFIX, childHash, true); + mTNByID[*ptr] = ptr; + d = ptr.get(); + filter->gotNode(true, childID, childHash, nodeData, ptr->getType()); + } + } + if (d) + { if (d->isInner() && !d->isFullBelow()) { have_all = false; stack.push(d); } } - catch (SHAMapMissingNode&) - { // node is not in the map + else + { have_all = false; ret.push_back(childHash); if (--max <= 0) @@ -263,7 +283,7 @@ SMAddNode SHAMap::addRootNode(const std::vector& rootNode, SHANod { Serializer s; root->addRaw(s, snfPREFIX); - filter->gotNode(*root, root->getNodeHash(), s.peekData(), root->getType()); + filter->gotNode(false, *root, root->getNodeHash(), s.peekData(), root->getType()); } return SMAddNode::useful(); @@ -299,7 +319,7 @@ SMAddNode SHAMap::addRootNode(const uint256& hash, const std::vectoraddRaw(s, snfPREFIX); - filter->gotNode(*root, root->getNodeHash(), s.peekData(), root->getType()); + filter->gotNode(false, *root, root->getNodeHash(), s.peekData(), root->getType()); } return SMAddNode::useful(); @@ -371,7 +391,7 @@ SMAddNode SHAMap::addKnownNode(const SHAMapNode& node, const std::vectoraddRaw(s, snfPREFIX); - filter->gotNode(node, hash, s.peekData(), newNode->getType()); + filter->gotNode(false, node, hash, s.peekData(), newNode->getType()); } mTNByID[*newNode] = newNode; diff --git a/src/cpp/ripple/SHAMapSync.h b/src/cpp/ripple/SHAMapSync.h index 1c010d104..ea7caee9d 100644 --- a/src/cpp/ripple/SHAMapSync.h +++ b/src/cpp/ripple/SHAMapSync.h @@ -13,7 +13,7 @@ class ConsensusTransSetSF : public SHAMapSyncFilter public: ConsensusTransSetSF() { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + virtual void gotNode(bool fromFilter, const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, SHAMapTreeNode::TNType); virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData); @@ -29,7 +29,7 @@ public: AccountStateSF(uint32 ledgerSeq) : mLedgerSeq(ledgerSeq) { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + virtual void gotNode(bool fromFilter, const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, SHAMapTreeNode::TNType) { theApp->getHashedObjectStore().store(hotACCOUNT_NODE, mLedgerSeq, nodeData, nodeHash); @@ -50,7 +50,7 @@ public: TransactionStateSF(uint32 ledgerSeq) : mLedgerSeq(ledgerSeq) { ; } - virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, + virtual void gotNode(bool fromFilter, const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, SHAMapTreeNode::TNType type) { theApp->getHashedObjectStore().store( diff --git a/src/cpp/ripple/TransactionAcquire.cpp b/src/cpp/ripple/TransactionAcquire.cpp index f2e1ee548..fb432afaf 100644 --- a/src/cpp/ripple/TransactionAcquire.cpp +++ b/src/cpp/ripple/TransactionAcquire.cpp @@ -195,9 +195,11 @@ SMAddNode TransactionAcquire::takeNodes(const std::list& nodeIDs, } } -void ConsensusTransSetSF::gotNode(const SHAMapNode& id, const uint256& nodeHash, +void ConsensusTransSetSF::gotNode(bool fromFilter, const SHAMapNode& id, const uint256& nodeHash, const std::vector& nodeData, SHAMapTreeNode::TNType type) { + if (fromFilter) + return; theApp->getTempNodeCache().store(nodeHash, nodeData); if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size() > 16)) { // this is a transaction, and we didn't have it diff --git a/src/cpp/ripple/Version.h b/src/cpp/ripple/Version.h index 897a19d97..46d70ff41 100644 --- a/src/cpp/ripple/Version.h +++ b/src/cpp/ripple/Version.h @@ -6,7 +6,7 @@ #define SERVER_VERSION_MAJOR 0 #define SERVER_VERSION_MINOR 8 -#define SERVER_VERSION_SUB "-b" +#define SERVER_VERSION_SUB "-c" #define SERVER_NAME "Ripple" #define SV_STRINGIZE(x) SV_STRINGIZE2(x) @@ -16,7 +16,7 @@ // Version we prefer to speak: #define PROTO_VERSION_MAJOR 1 -#define PROTO_VERSION_MINOR 2 +#define PROTO_VERSION_MINOR 3 // Version we will speak to: #define MIN_PROTO_MAJOR 1 diff --git a/src/cpp/ripple/ripple.proto b/src/cpp/ripple/ripple.proto index d35640ba1..3046b9087 100644 --- a/src/cpp/ripple/ripple.proto +++ b/src/cpp/ripple/ripple.proto @@ -119,6 +119,8 @@ message TMStatusChange { optional bytes ledgerHash = 4; optional bytes ledgerHashPrevious = 5; optional uint64 networkTime = 6; + optional uint32 firstSeq = 7; + optional uint32 lastSeq = 8; }