From 30dd46d8124bffdf2219919f5db8d4cf8e2b909d Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 02:50:04 -0700 Subject: [PATCH 01/14] Improve SHAMapMissingNode --- src/SHAMap.cpp | 18 ++++++++++++------ src/SHAMap.h | 14 ++++++++++++-- src/SHAMapSync.cpp | 7 ++++--- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 5dd31f5651..533b9d6710 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -93,7 +93,8 @@ std::stack SHAMap::getStack(const uint256& id, bool inc { if (partialOk) return stack; - throw mn; + mn.setTargetNode(id); + throw; } } @@ -153,10 +154,15 @@ SHAMapTreeNode::pointer SHAMap::walkTo(const uint256& id, bool modify) return inNode; uint256 childHash = inNode->getChildHash(branch); - SHAMapTreeNode::pointer nextNode = getNode(inNode->getChildNodeID(branch), childHash, false); - if (!nextNode) - throw SHAMapMissingNode(inNode->getChildNodeID(branch), childHash); - inNode = nextNode; + try + { + inNode = getNode(inNode->getChildNodeID(branch), childHash, false); + } + catch (SHAMapMissingNode& mn) + { + mn.setTargetNode(id); + throw; + } } if (inNode->getTag() != id) return SHAMapTreeNode::pointer(); @@ -175,7 +181,7 @@ SHAMapTreeNode* SHAMap::walkToPointer(const uint256& id) if (nextHash.isZero()) return NULL; inNode = getNodePointer(inNode->getChildNodeID(branch), nextHash); if (!inNode) - throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash); + throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash, id); } return (inNode->getTag() == id) ? inNode : NULL; } diff --git a/src/SHAMap.h b/src/SHAMap.h index c239bad307..a942943b1a 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -241,16 +241,26 @@ class SHAMapMissingNode : public std::runtime_error protected: SHAMapNode mNodeID; uint256 mNodeHash; + uint256 mTargetIndex; public: SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash) : std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash) { ; } + + SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash, const uint256& targetIndex) : + std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash), mTargetIndex(targetIndex) + { ; } + virtual ~SHAMapMissingNode() throw() { ; } - const SHAMapNode& getNodeID() const { return mNodeID; } - const uint256& getNodeHash() const { return mNodeHash; } + void setTargetNode(const uint256& tn) { mTargetIndex = tn; } + + const SHAMapNode& getNodeID() const { return mNodeID; } + const uint256& getNodeHash() const { return mNodeHash; } + const uint256& getTargetIndex() const { return mTargetIndex; } + bool hasTargetIndex() const { return !mTargetIndex.isZero(); } }; class SHAMap diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index fef3cf04ad..baf7de9501 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -53,7 +53,7 @@ void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vectorisInner() && !nextNode->isFullBelow()) return true; } - catch (SHAMapMissingNode) + catch (SHAMapMissingNode&) { return true; } } iNode->setFullBelow(); } while (!stack.empty()); - if (root->isFullBelow()) clearSynching(); + if (root->isFullBelow()) + clearSynching(); return true; } From 67477beecb484d35ef8a3bc33d542310bdd78b02 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 03:13:28 -0700 Subject: [PATCH 02/14] Track the type of map in the SHAMap. This will make it easier to parse missing nodes at ledger level. --- src/Ledger.cpp | 8 ++++---- src/LedgerConsensus.cpp | 4 ++-- src/SHAMap.cpp | 8 ++++---- src/SHAMap.h | 13 +++++++++++-- src/SHAMapSync.cpp | 2 +- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index f52e1f1e71..35ffc7dcfe 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -24,7 +24,7 @@ SETUP_LOG(); Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(startAmount), mLedgerSeq(1), mCloseTime(0), mParentCloseTime(0), mCloseResolution(LEDGER_TIME_ACCURACY), mCloseFlags(0), mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false), - mTransactionMap(new SHAMap()), mAccountStateMap(new SHAMap()) + mTransactionMap(new SHAMap(smtTRANSACTION)), mAccountStateMap(new SHAMap(smtFREE)) { // special case: put coins in root account AccountState::pointer startAccount = boost::make_shared(masterID); @@ -62,7 +62,7 @@ Ledger::Ledger(bool /* dummy */, Ledger& prevLedger) : mTotCoins(prevLedger.mTotCoins), mLedgerSeq(prevLedger.mLedgerSeq + 1), mParentCloseTime(prevLedger.mCloseTime), mCloseResolution(prevLedger.mCloseResolution), mCloseFlags(0), mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false), - mTransactionMap(new SHAMap()), mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true)) + mTransactionMap(new SHAMap(smtTRANSACTION)), mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true)) { // Create a new ledger that follows this one prevLedger.updateHash(); mParentHash = prevLedger.getHash(); @@ -123,8 +123,8 @@ void Ledger::setRaw(const Serializer &s) updateHash(); if(mValidHash) { - mTransactionMap = boost::make_shared(mTransHash); - mAccountStateMap = boost::make_shared(mAccountHash); + mTransactionMap = boost::make_shared(smtTRANSACTION, mTransHash); + mAccountStateMap = boost::make_shared(smtSTATE, mAccountHash); } } diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index a07965ae6b..c915faad7d 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -27,7 +27,7 @@ SETUP_LOG(); TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) { - mMap = boost::make_shared(hash); + mMap = boost::make_shared(smtTRANSACTION, hash); } void TransactionAcquire::done() @@ -759,7 +759,7 @@ SHAMap::pointer LedgerConsensus::getTransactionTree(const uint256& hash, bool do { if (!hash) { - SHAMap::pointer empty = boost::make_shared(); + SHAMap::pointer empty = boost::make_shared(smtTRANSACTION); mapComplete(hash, empty, false); return empty; } diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 533b9d6710..97580186ea 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -41,14 +41,14 @@ std::size_t hash_value(const uint160& u) } -SHAMap::SHAMap(uint32 seq) : mSeq(seq), mState(smsModifying) +SHAMap::SHAMap(SHAMapType t, uint32 seq) : mSeq(seq), mState(smsModifying), mType(t) { root = boost::make_shared(mSeq, SHAMapNode(0, uint256())); root->makeInner(); mTNByID[*root] = root; } -SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(smsSynching) +SHAMap::SHAMap(SHAMapType t, const uint256& hash) : mSeq(0), mState(smsSynching), mType(t) { // FIXME: Need to acquire root node root = boost::make_shared(mSeq, SHAMapNode(0, uint256())); root->makeInner(); @@ -58,7 +58,7 @@ SHAMap::SHAMap(const uint256& hash) : mSeq(0), mState(smsSynching) SHAMap::pointer SHAMap::snapShot(bool isMutable) { // Return a new SHAMap that is an immutable snapshot of this one // Initially nodes are shared, but CoW is forced on both ledgers - SHAMap::pointer ret = boost::make_shared(); + SHAMap::pointer ret = boost::make_shared(mType); SHAMap& newMap = *ret; newMap.mSeq = ++mSeq; newMap.mTNByID = mTNByID; @@ -808,7 +808,7 @@ BOOST_AUTO_TEST_CASE( SHAMap_test ) h4.SetHex("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8"); h5.SetHex("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); - SHAMap sMap; + SHAMap sMap(smtFREE); SHAMapItem i1(h1, IntToVUC(1)), i2(h2, IntToVUC(2)), i3(h3, IntToVUC(3)), i4(h4, IntToVUC(4)), i5(h5, IntToVUC(5)); if (!sMap.addItem(i2, true, false)) BOOST_FAIL("no add"); diff --git a/src/SHAMap.h b/src/SHAMap.h index a942943b1a..11c34a74d2 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -128,6 +128,13 @@ enum SHANodeFormat snfWIRE = 2, // Compressed form used on the wire }; +enum SHAMapType +{ + smtTRANSACTION =1, // A tree of transactions + smtSTATE =2, // A tree of state nodes + smtFREE =3, // A tree not part of a ledger +}; + class SHAMapTreeNode : public SHAMapNode { friend class SHAMap; @@ -282,6 +289,8 @@ private: SHAMapState mState; + SHAMapType mType; + protected: void dirtyUp(std::stack& stack, const uint256& target, uint256 prevHash); @@ -306,8 +315,8 @@ protected: public: // build new map - SHAMap(uint32 seq = 0); - SHAMap(const uint256& hash); + SHAMap(SHAMapType t, uint32 seq = 0); + SHAMap(SHAMapType t, const uint256& hash); ~SHAMap() { mState = smsInvalid; } diff --git a/src/SHAMapSync.cpp b/src/SHAMapSync.cpp index baf7de9501..7fdfac494c 100644 --- a/src/SHAMapSync.cpp +++ b/src/SHAMapSync.cpp @@ -421,7 +421,7 @@ BOOST_AUTO_TEST_CASE( SHAMapSync_test ) srand(seed); cLog(lsTRACE) << "Constructing maps"; - SHAMap source, destination; + SHAMap source(smtFREE), destination(smtFREE); // add random data to the source map cLog(lsTRACE) << "Adding random data"; From 4f1ebc28840f19a4465ce09969285543a4173e9e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 03:16:18 -0700 Subject: [PATCH 03/14] Put the missing node type in the SHAMapMissingNode exception. --- src/SHAMap.cpp | 8 ++++---- src/SHAMap.h | 11 +++++++---- src/SHAMapDiff.cpp | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 97580186ea..efd97e2668 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -181,7 +181,7 @@ SHAMapTreeNode* SHAMap::walkToPointer(const uint256& id) if (nextHash.isZero()) return NULL; inNode = getNodePointer(inNode->getChildNodeID(branch), nextHash); if (!inNode) - throw SHAMapMissingNode(inNode->getChildNodeID(branch), nextHash, id); + throw SHAMapMissingNode(mType, inNode->getChildNodeID(branch), nextHash, id); } return (inNode->getTag() == id) ? inNode : NULL; } @@ -686,11 +686,11 @@ void SHAMapItem::dump() SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash) { if (!theApp->running()) - throw SHAMapMissingNode(id, hash); + throw SHAMapMissingNode(mType, id, hash); HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); if (!obj) - throw SHAMapMissingNode(id, hash); + throw SHAMapMissingNode(mType, id, hash); assert(Serializer::getSHA512Half(obj->getData()) == hash); try @@ -704,7 +704,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui catch (...) { cLog(lsWARNING) << "fetchNodeExternal gets an invalid node: " << hash; - throw SHAMapMissingNode(id, hash); + throw SHAMapMissingNode(mType, id, hash); } } diff --git a/src/SHAMap.h b/src/SHAMap.h index 11c34a74d2..72a4f23c3e 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -246,17 +246,19 @@ public: class SHAMapMissingNode : public std::runtime_error { protected: + SHAMapType mType; SHAMapNode mNodeID; uint256 mNodeHash; uint256 mTargetIndex; public: - SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash) : - std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash) + SHAMapMissingNode(SHAMapType t, const SHAMapNode& nodeID, const uint256& nodeHash) : + std::runtime_error(nodeID.getString()), mType(t), mNodeID(nodeID), mNodeHash(nodeHash) { ; } - SHAMapMissingNode(const SHAMapNode& nodeID, const uint256& nodeHash, const uint256& targetIndex) : - std::runtime_error(nodeID.getString()), mNodeID(nodeID), mNodeHash(nodeHash), mTargetIndex(targetIndex) + SHAMapMissingNode(SHAMapType t, const SHAMapNode& nodeID, const uint256& nodeHash, const uint256& targetIndex) : + std::runtime_error(nodeID.getString()), mType(t), + mNodeID(nodeID), mNodeHash(nodeHash), mTargetIndex(targetIndex) { ; } virtual ~SHAMapMissingNode() throw() @@ -264,6 +266,7 @@ public: void setTargetNode(const uint256& tn) { mTargetIndex = tn; } + SHAMapType getMapType() const { return mType; } const SHAMapNode& getNodeID() const { return mNodeID; } const uint256& getNodeHash() const { return mNodeHash; } const uint256& getTargetIndex() const { return mTargetIndex; } diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index fa1ae6a642..d0e78e8f98 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -120,7 +120,7 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount if (!ourNode || !otherNode) { assert(false); - throw SHAMapMissingNode(dNode.mNodeID, uint256()); + throw SHAMapMissingNode(mType, dNode.mNodeID, uint256()); } if (ourNode->isLeaf() && otherNode->isLeaf()) From c43934ab6f2d329386c15042c5865c915e502c3f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 05:06:24 -0700 Subject: [PATCH 04/14] Fix lots of places I used 'empty' instead of 'clear'. This is why I like 'isEmpty'. --- src/Application.cpp | 2 +- src/LedgerAcquire.cpp | 2 +- src/PubKeyCache.cpp | 2 +- src/SerializedObject.cpp | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index a5d458dc18..fa8861fa13 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -215,8 +215,8 @@ void Application::loadOldLedger() std::cout << "No Ledger found?" << std::endl; exit(-1); } - lastLedger->setClosed(); + Ledger::pointer openLedger = boost::make_shared(false, boost::ref(*lastLedger)); mMasterLedger.switchLedgers(lastLedger, openLedger); mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC()); diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index d9056fdad4..1eac7dc3f8 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -124,7 +124,7 @@ void LedgerAcquire::done() setComplete(); mLock.lock(); triggers = mOnComplete; - mOnComplete.empty(); + mOnComplete.clear(); mLock.unlock(); if (mLedger) diff --git a/src/PubKeyCache.cpp b/src/PubKeyCache.cpp index 503fefc3d1..4ca514039e 100644 --- a/src/PubKeyCache.cpp +++ b/src/PubKeyCache.cpp @@ -69,6 +69,6 @@ CKey::pointer PubKeyCache::store(const NewcoinAddress& id, const CKey::pointer& void PubKeyCache::clear() { boost::mutex::scoped_lock sl(mLock); - mCache.empty(); + mCache.clear(); } // vim:ts=4 diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index fb30248f64..8e6e9df605 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -120,8 +120,8 @@ std::auto_ptr STObject::makeDeserializedObject(SerializedTypeID void STObject::set(const std::vector& type) { - mData.empty(); - mType.empty(); + mData.clear(); + mType.clear(); BOOST_FOREACH(const SOElement::ptr& elem, type) { @@ -138,7 +138,7 @@ bool STObject::setType(const std::vector &type) boost::ptr_vector newData; bool valid = true; - mType.empty(); + mType.clear(); BOOST_FOREACH(const SOElement::ptr& elem, type) { bool match = false; @@ -204,7 +204,7 @@ bool STObject::isFieldAllowed(SField::ref field) bool STObject::set(SerializerIterator& sit, int depth) { // return true = terminated with end-of-object - mData.empty(); + mData.clear(); while (!sit.empty()) { int type, field; From 61c9d30732948c84900bf2c65c7558601e623021 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 05:13:29 -0700 Subject: [PATCH 05/14] Set up to fix the bug Jed reported and also support faulting in nodes under ledgers. --- src/Ledger.cpp | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 35ffc7dcfe..737579df73 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -24,7 +24,8 @@ SETUP_LOG(); Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(startAmount), mLedgerSeq(1), mCloseTime(0), mParentCloseTime(0), mCloseResolution(LEDGER_TIME_ACCURACY), mCloseFlags(0), mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false), - mTransactionMap(new SHAMap(smtTRANSACTION)), mAccountStateMap(new SHAMap(smtFREE)) + mTransactionMap(boost::make_shared(smtTRANSACTION)), + mAccountStateMap(boost::make_shared(smtSTATE)) { // special case: put coins in root account AccountState::pointer startAccount = boost::make_shared(masterID); @@ -42,9 +43,19 @@ Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint25 : mParentHash(parentHash), mTransHash(transHash), mAccountHash(accountHash), mTotCoins(totCoins), mLedgerSeq(ledgerSeq), mCloseTime(closeTime), mParentCloseTime(parentCloseTime), mCloseResolution(closeResolution), mCloseFlags(closeFlags), - mClosed(false), mValidHash(false), mAccepted(false), mImmutable(isMutable) + mClosed(false), mValidHash(false), mAccepted(false), mImmutable(isMutable), + mTransactionMap(boost::make_shared(smtTRANSACTION)), + mAccountStateMap(boost::make_shared(smtSTATE)) { updateHash(); + if (mTransHash.isNonZero()) + { + // WRITEME + } + if (mAccountHash.isNonZero()) + { + // WRITEME + } } Ledger::Ledger(Ledger& ledger, bool isMutable) : mTotCoins(ledger.mTotCoins), mLedgerSeq(ledger.mLedgerSeq), @@ -62,7 +73,8 @@ Ledger::Ledger(bool /* dummy */, Ledger& prevLedger) : mTotCoins(prevLedger.mTotCoins), mLedgerSeq(prevLedger.mLedgerSeq + 1), mParentCloseTime(prevLedger.mCloseTime), mCloseResolution(prevLedger.mCloseResolution), mCloseFlags(0), mClosed(false), mValidHash(false), mAccepted(false), mImmutable(false), - mTransactionMap(new SHAMap(smtTRANSACTION)), mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true)) + mTransactionMap(boost::make_shared(smtTRANSACTION)), + mAccountStateMap(prevLedger.mAccountStateMap->snapShot(true)) { // Create a new ledger that follows this one prevLedger.updateHash(); mParentHash = prevLedger.getHash(); From 8c790b1d8824c44f5a3288985fc2c744fb428360 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 05:17:35 -0700 Subject: [PATCH 06/14] Remove dead code. --- src/NetworkOPs.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 855128bdf2..6a01d4268f 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -585,7 +585,6 @@ bool NetworkOPs::checkLastClosedLedger(const std::vector& peerLis if (!mAcquiringLedger->isComplete()) { // add more peers int count = 0; - std::vector peers=theApp->getConnectionPool().getPeerVector(); BOOST_FOREACH(Peer::ref it, peerList) { if (it->getClosedLedgerHash() == closedLedger) From 07cd8ad9bd22c605cda0b6bca7df384a31456962 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 06:04:06 -0700 Subject: [PATCH 07/14] Some extra debug on the dirty node flush code. It appears good. --- src/Ledger.cpp | 9 +++++---- src/SHAMap.cpp | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 737579df73..92f5621543 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -340,10 +340,11 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger) ledger->mAccountHash.GetHex() % ledger->mTransHash.GetHex())); // write out dirty nodes - while(ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq)) - { ; } - while(ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq)) - { ; } + int fc; + while ((fc = ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq)) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; } + while ((fc = ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq)) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } ledger->disarmDirty(); SHAMap& txSet = *ledger->peekTransactionMap(); diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index efd97e2668..3d6f706202 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -725,6 +725,8 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end()) { + tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first; + tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first; s.erase(); it->second->addRaw(s, snfPREFIX); theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half()); From 55e38c40f72a9e7e60c54fa4bf69cb840c602bf9 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:01:03 -0700 Subject: [PATCH 08/14] Cleanups. --- src/SHAMap.cpp | 10 ++++++++++ src/SHAMap.h | 3 +++ 2 files changed, 13 insertions(+) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 3d6f706202..09877fa403 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -690,7 +690,10 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); if (!obj) + { + Log(lsTRACE) << "fetchNodeExternal: missing " << hash; throw SHAMapMissingNode(mType, id, hash); + } assert(Serializer::getSHA512Half(obj->getData()) == hash); try @@ -708,6 +711,13 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui } } +void SHAMap::fetchRoot(const uint256& hash) +{ + root = fetchNodeExternal(SHAMapNode(), hash); + root->makeInner(); + mTNByID[*root] = root; +} + void SHAMap::armDirty() { // begin saving dirty nodes ++mSeq; diff --git a/src/SHAMap.h b/src/SHAMap.h index 72a4f23c3e..f144b5ca9f 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -273,6 +273,8 @@ public: bool hasTargetIndex() const { return !mTargetIndex.isZero(); } }; +extern std::ostream& operator<<(std::ostream&, const SHAMapMissingNode&); + class SHAMap { public: @@ -330,6 +332,7 @@ public: ScopedLock Lock() const { return ScopedLock(mLock); } bool hasNode(const SHAMapNode& id); + void fetchRoot(const uint256& hash); // normal hash access functions bool hasItem(const uint256& id); From cf6206f19b2a52e5b69d2f2c2403b6c500c22ab4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:01:23 -0700 Subject: [PATCH 09/14] Some nicer logging. --- src/SHAMapNodes.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 746176aba3..ad1e27749f 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -18,6 +18,9 @@ std::string SHAMapNode::getString() const { + if ((mDepth == 0) && (mNodeID.isZero())) + return "NodeID(root)"; + return str(boost::format("NodeID(%s,%s)") % boost::lexical_cast(mDepth) % mNodeID.GetHex()); @@ -507,4 +510,16 @@ bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash) return updateHash(); } +std::ostream& operator<<(std::ostream& out, const SHAMapMissingNode& mn) +{ + if (mn.getMapType() == smtTRANSACTION) + out << "Missing/TXN(" << mn.getNodeID() << ")"; + else if (mn.getMapType() == smtSTATE) + out << "Missing/STA(" << mn.getNodeID() << ")"; + else + out << "Missing/" << mn.getNodeID(); +} + + + // vim:ts=4 From e18f8c47f5ce7a94e37a71e0ddc34c1525c79305 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:01:40 -0700 Subject: [PATCH 10/14] Use the local hashed object store to jump start fetching a ledger. --- src/LedgerAcquire.cpp | 39 +++++++++++++++++++++++++++++++++++++++ src/LedgerAcquire.h | 1 + 2 files changed, 40 insertions(+) diff --git a/src/LedgerAcquire.cpp b/src/LedgerAcquire.cpp index 1eac7dc3f8..95cf414156 100644 --- a/src/LedgerAcquire.cpp +++ b/src/LedgerAcquire.cpp @@ -96,6 +96,45 @@ LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE #endif } +bool LedgerAcquire::tryLocal() +{ // return value: true = no more work to do + HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(mHash); + if (!node) + return false; + + mLedger = boost::make_shared(strCopy(node->getData())); + assert(mLedger->getHash() == mHash); + mHaveBase = true; + + if (!mLedger->getTransHash()) + mHaveTransactions = true; + else + { + try + { + mLedger->peekTransactionMap()->fetchRoot(mLedger->getTransHash()); + } + catch (SHAMapMissingNode&) + { + } + } + + if (!mLedger->getAccountHash()) + mHaveState = true; + else + { + try + { + mLedger->peekAccountStateMap()->fetchRoot(mLedger->getAccountHash()); + } + catch (SHAMapMissingNode&) + { + } + } + + return mHaveTransactions && mHaveState; +} + void LedgerAcquire::onTimer() { if (getTimeouts() > 6) diff --git a/src/LedgerAcquire.h b/src/LedgerAcquire.h index b589618299..7e2da168d8 100644 --- a/src/LedgerAcquire.h +++ b/src/LedgerAcquire.h @@ -94,6 +94,7 @@ public: bool takeAsNode(const std::list& IDs, const std::list >& data); bool takeAsRootNode(const std::vector& data); void trigger(Peer::ref, bool timer); + bool tryLocal(); }; class LedgerAcquireMaster From 4a8668557447485bf972609da140eb971ca3d253 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:02:02 -0700 Subject: [PATCH 11/14] A ledger fetched from SQL is always immutable. --- src/Ledger.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ledger.h b/src/Ledger.h index 41e30d2dbf..ffb9dbd81e 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -93,7 +93,7 @@ public: Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash, uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution, - uint32 ledgerSeq, bool immutable); // used for database ledgers + uint32 ledgerSeq); // used for database ledgers Ledger(const std::vector& rawLedger); From 37f246396fc4bf4861104c23787c86de5cab3b01 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:02:25 -0700 Subject: [PATCH 12/14] Try to populate the ledger root. --- src/Ledger.cpp | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 92f5621543..53491b9915 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -39,23 +39,21 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(s } Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash, - uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution, uint32 ledgerSeq,bool isMutable) + uint64 totCoins, uint32 closeTime, uint32 parentCloseTime, int closeFlags, int closeResolution, uint32 ledgerSeq) : mParentHash(parentHash), mTransHash(transHash), mAccountHash(accountHash), mTotCoins(totCoins), mLedgerSeq(ledgerSeq), mCloseTime(closeTime), mParentCloseTime(parentCloseTime), mCloseResolution(closeResolution), mCloseFlags(closeFlags), - mClosed(false), mValidHash(false), mAccepted(false), mImmutable(isMutable), - mTransactionMap(boost::make_shared(smtTRANSACTION)), - mAccountStateMap(boost::make_shared(smtSTATE)) + mClosed(false), mValidHash(false), mAccepted(false), mImmutable(true), + mTransactionMap(boost::make_shared(smtTRANSACTION, transHash)), + mAccountStateMap(boost::make_shared(smtSTATE, accountHash)) { updateHash(); if (mTransHash.isNonZero()) - { - // WRITEME - } + mTransactionMap->fetchRoot(mTransHash); if (mAccountHash.isNonZero()) - { - // WRITEME - } + mAccountStateMap->fetchRoot(mAccountHash); + mTransactionMap->setImmutable(); + mAccountStateMap->setImmutable(); } Ledger::Ledger(Ledger& ledger, bool isMutable) : mTotCoins(ledger.mTotCoins), mLedgerSeq(ledger.mLedgerSeq), @@ -438,7 +436,7 @@ Ledger::pointer Ledger::getSQL(const std::string& sql) } Ledger::pointer ret = Ledger::pointer(new Ledger(prevHash, transHash, accountHash, totCoins, - closingTime, prevClosingTime, closeFlags, closeResolution, ledgerSeq, true)); + closingTime, prevClosingTime, closeFlags, closeResolution, ledgerSeq)); if (ret->getHash() != ledgerHash) { if (sLog(lsERROR)) From ac8e2292cf7bed6e7b48193932aa751248bcafcf Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:03:14 -0700 Subject: [PATCH 13/14] Add a comment. --- src/Ledger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 53491b9915..5be6eb02b2 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -46,7 +46,7 @@ Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint25 mClosed(false), mValidHash(false), mAccepted(false), mImmutable(true), mTransactionMap(boost::make_shared(smtTRANSACTION, transHash)), mAccountStateMap(boost::make_shared(smtSTATE, accountHash)) -{ +{ // This will throw if the root nodes are not available locally updateHash(); if (mTransHash.isNonZero()) mTransactionMap->fetchRoot(mTransHash); From 2a13b9b7db8b45950373dd6bf5bf8b91dfd8edd8 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 11 Oct 2012 07:03:21 -0700 Subject: [PATCH 14/14] Fix the '--load' logic. --- src/Application.cpp | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/Application.cpp b/src/Application.cpp index fa8861fa13..11b19895b3 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -208,17 +208,25 @@ void Application::startNewLedger() void Application::loadOldLedger() { - Ledger::pointer lastLedger = Ledger::getSQL("SELECT * from Ledgers order by LedgerSeq desc limit 1;"); - - if (!lastLedger) + try { - std::cout << "No Ledger found?" << std::endl; + Ledger::pointer lastLedger = Ledger::getSQL("SELECT * from Ledgers order by LedgerSeq desc limit 1;"); + + if (!lastLedger) + { + std::cout << "No Ledger found?" << std::endl; + exit(-1); + } + lastLedger->setClosed(); + + Ledger::pointer openLedger = boost::make_shared(false, boost::ref(*lastLedger)); + mMasterLedger.switchLedgers(lastLedger, openLedger); + mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC()); + } + catch (SHAMapMissingNode& mn) + { + Log(lsFATAL) << "Cannot load ledger. " << mn; exit(-1); } - lastLedger->setClosed(); - - Ledger::pointer openLedger = boost::make_shared(false, boost::ref(*lastLedger)); - mMasterLedger.switchLedgers(lastLedger, openLedger); - mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC()); } // vim:ts=4