From 0030845572fe85431ef44ca785c302fbaa693d72 Mon Sep 17 00:00:00 2001 From: jed Date: Mon, 15 Oct 2012 13:49:41 -0700 Subject: [PATCH 01/27] windows --- newcoin.vcxproj | 10 +++++----- newcoin.vcxproj.filters | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/newcoin.vcxproj b/newcoin.vcxproj index b6093c53a0..bb2b464845 100644 --- a/newcoin.vcxproj +++ b/newcoin.vcxproj @@ -268,13 +268,13 @@ Designer - - Document - ..\protoc-2.4.1-win32\protoc -I=..\newcoin\src --cpp_out=..\newcoin\obj\src ..\newcoin\src\newcoin.proto - obj\src\newcoin.pb.h - + + Document + d:/code/protoc-2.4.1-win32/protoc -I=..\newcoin --cpp_out=ripple ..\newcoin/src/ripple.proto + D:\code\newcoin\obj\src\ripple.pb.h + diff --git a/newcoin.vcxproj.filters b/newcoin.vcxproj.filters index b638955b76..2c3a38dac3 100644 --- a/newcoin.vcxproj.filters +++ b/newcoin.vcxproj.filters @@ -567,8 +567,12 @@ + + + + - + \ No newline at end of file From ac60bfbd437999864dff9d830b3a0aa5f9ffd4c3 Mon Sep 17 00:00:00 2001 From: jed Date: Mon, 15 Oct 2012 16:15:07 -0700 Subject: [PATCH 02/27] windows --- newcoin.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newcoin.vcxproj b/newcoin.vcxproj index bb2b464845..d83a20bd48 100644 --- a/newcoin.vcxproj +++ b/newcoin.vcxproj @@ -272,7 +272,7 @@ Document - d:/code/protoc-2.4.1-win32/protoc -I=..\newcoin --cpp_out=ripple ..\newcoin/src/ripple.proto + d:/code/protoc-2.4.1-win32/protoc -I=..\newcoin --cpp_out=D:\code\newcoin\obj\ ..\newcoin/src/ripple.proto D:\code\newcoin\obj\src\ripple.pb.h From b6dc333b68e491d89e4ca58dd1c7cd88fc03263e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 03:52:46 -0700 Subject: [PATCH 03/27] Add a 'sort' function to an STArray. They can be used in cases where order is significant or where they're sorted. --- src/SerializedObject.cpp | 5 +++++ src/SerializedObject.h | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index 678b1add3c..c8d243952e 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -866,6 +866,11 @@ STArray* STArray::construct(SerializerIterator& sit, SField::ref field) return new STArray(field, value); } +void STArray::sort(bool (*compare)(const STObject&, const STObject&)) +{ + std::sort(value.begin(), value.end(), compare); +} + std::auto_ptr STObject::parseJson(const Json::Value& object, SField::ref inName, int depth) { if (!object.isObject()) diff --git a/src/SerializedObject.h b/src/SerializedObject.h index 7ecd539eb3..f2c389f09d 100644 --- a/src/SerializedObject.h +++ b/src/SerializedObject.h @@ -152,9 +152,9 @@ class STArray : public SerializedType { public: typedef std::vector vector; - typedef std::vector::iterator iterator; + typedef std::vector::iterator iterator; typedef std::vector::const_iterator const_iterator; - typedef std::vector::reverse_iterator reverse_iterator; + typedef std::vector::reverse_iterator reverse_iterator; typedef std::vector::const_reverse_iterator const_reverse_iterator; typedef std::vector::size_type size_type; @@ -205,6 +205,8 @@ public: virtual Json::Value getJson(int) const; virtual void add(Serializer& s) const; + void sort(bool (*compare)(const STObject& o1, const STObject& o2)); + bool operator==(const STArray &s) { return value == s.value; } bool operator!=(const STArray &s) { return value != s.value; } From 1f58c4326dd7806a3cc5b050eab89dda2af354b1 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 05:05:15 -0700 Subject: [PATCH 04/27] In consensus process, if we don't have the LCL, only *de*mote to tracking. Don't promote to tracking. --- src/NetworkOPs.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index c594c086f1..af155ae692 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -646,8 +646,11 @@ int NetworkOPs::beginConsensus(const uint256& networkClosed, Ledger::pointer clo Ledger::pointer prevLedger = mLedgerMaster->getLedgerByHash(closingLedger->getParentHash()); if (!prevLedger) { // this shouldn't happen unless we jump ledgers - cLog(lsWARNING) << "Don't have LCL, going to tracking"; - setMode(omTRACKING); + if (mMode == omFULL) + { + cLog(lsWARNING) << "Don't have LCL, going to tracking"; + setMode(omTRACKING); + } return 3; } assert(prevLedger->getHash() == closingLedger->getParentHash()); From 84ef7dff0f1b89885be80d34e8153f90d0c573be Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 05:05:36 -0700 Subject: [PATCH 05/27] Support for the new transaction meta types. --- src/SerializeProto.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/SerializeProto.h b/src/SerializeProto.h index df7d2f6d5c..e3b594db7c 100644 --- a/src/SerializeProto.h +++ b/src/SerializeProto.h @@ -25,6 +25,7 @@ // 8-bit integers FIELD(CloseResolution, UINT8, 1) FIELD(TemplateEntryType, UINT8, 2) + FIELD(TransactionResult, UINT8, 3) // 16-bit integers FIELD(LedgerEntryType, UINT16, 1) @@ -92,8 +93,14 @@ FIELD(SendMax, AMOUNT, 9) // currency amount (uncommon) - FIELD(MinimumOffer, AMOUNT, 16) - FIELD(RippleEscrow, AMOUNT, 17) + FIELD(MinimumOffer, AMOUNT, 16) + FIELD(RippleEscrow, AMOUNT, 17) + FIELD(PreviousBalance, AMOUNT, 18) + FIELD(FinalBalance, AMOUNT, 19) + FIELD(PreviousTakerPays, AMOUNT, 20) + FIELD(PreviousTakerGets, AMOUNT, 21) + FIELD(FinalTakerPays, AMOUNT, 22) + FIELD(FinalTakerGets, AMOUNT, 23) // variable length FIELD(PublicKey, VL, 1) @@ -116,6 +123,11 @@ FIELD(Target, ACCOUNT, 7) FIELD(AuthorizedKey, ACCOUNT, 8) + // account (uncommon) + FIELD(PreviousAccount, ACCOUNT, 16) + FIELD(LowID, ACCOUNT, 17) + FIELD(HighID, ACCOUNT, 18) + // path set FIELD(Paths, PATHSET, 1) @@ -125,9 +137,13 @@ // inner object // OBJECT/1 is reserved for end of object FIELD(TemplateEntry, OBJECT, 1) + FIELD(CreatedNode, OBJECT, 2) + FIELD(DeletedNode, OBJECT, 3) + FIELD(ModifiedNode, OBJECT, 4) // array of objects // ARRAY/1 is reserved for end of array + FIELD(TransactionMetaData, ARRAY, 1) FIELD(SigningAccounts, ARRAY, 2) FIELD(TxnSignatures, ARRAY, 3) FIELD(Signatures, ARRAY, 4) From 94d607def0d85a9aaa50f82edc90d7c56ac2759a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 05:27:48 -0700 Subject: [PATCH 06/27] Move deferred proposals to network ops. Include the last closed ledger in proposals. Call handleLCL even if we happen to jump our notion of the network's last closed ledger to the ledger we had as our locally last-closed ledger. --- src/LedgerConsensus.cpp | 29 ++++++++++++++--------------- src/LedgerConsensus.h | 7 ------- src/NetworkOPs.h | 5 ++++- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 83dcbefefe..bb56dcde9c 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -259,7 +259,7 @@ void LedgerConsensus::checkLCL() uint256 netLgr = mPrevLedgerHash; int netLgrCount = 0; - uint256 favoredLedger = (mState == lcsPRE_CLOSE) ? uint256() : mPrevLedgerHash; // Don't get stuck one ledger behind + uint256 favoredLedger = (mState == lcsPRE_CLOSE) ? uint256() : mPrevLedgerHash; // Don't get stuck one ledger back boost::unordered_map vals = theApp->getValidations().getCurrentValidations(favoredLedger); @@ -287,15 +287,18 @@ void LedgerConsensus::checkLCL() << status << ", " << (mHaveCorrectLCL ? "CorrectLCL" : "IncorrectLCL"); cLog(lsWARNING) << mPrevLedgerHash << " to " << netLgr; -#ifdef DEBUG - BOOST_FOREACH(u256_cvc_pair& it, vals) - cLog(lsDEBUG) << "V: " << it.first << ", " << it.second.first; -#endif + if (sLog(lsDEBUG)) + { + BOOST_FOREACH(u256_cvc_pair& it, vals) + cLog(lsDEBUG) << "V: " << it.first << ", " << it.second.first; + } if (mHaveCorrectLCL) theApp->getOPs().consensusViewChange(); handleLCL(netLgr); } + else if (mPreviousLedger->getHash() != mPrevLedgerHash) + handleLCL(netLgr); } void LedgerConsensus::handleLCL(const uint256& lclHash) @@ -810,7 +813,9 @@ void LedgerConsensus::propose() { cLog(lsTRACE) << "We propose: " << mOurPosition->getCurrentHash(); ripple::TMProposeSet prop; + prop.set_currenttxhash(mOurPosition->getCurrentHash().begin(), 256 / 8); + prop.set_previousledger(mOurPosition->getPrevLedger().begin(), 256 / 8); prop.set_proposeseq(mOurPosition->getProposeSeq()); prop.set_closetime(mOurPosition->getCloseTime()); @@ -958,18 +963,12 @@ void LedgerConsensus::Saccept(boost::shared_ptr This, SHAMap::p This->accept(txSet); } -void LedgerConsensus::deferProposal(const LedgerProposal::pointer& proposal, const NewcoinAddress& peerPublic) -{ - std::list& props = mDeferredProposals[peerPublic.getNodeID()]; - if (props.size() >= (mPreviousProposers + 10)) - props.pop_front(); - props.push_back(proposal); -} - void LedgerConsensus::playbackProposals() { + boost::unordered_map >& storedProposals = theApp->getOPs().peekStoredProposals(); for (boost::unordered_map< uint160, std::list >::iterator - it = mDeferredProposals.begin(), end = mDeferredProposals.end(); it != end; ++it) + it = storedProposals.begin(), end = storedProposals.end(); it != end; ++it) { BOOST_FOREACH(const LedgerProposal::pointer& proposal, it->second) { @@ -978,7 +977,7 @@ void LedgerConsensus::playbackProposals() proposal->setPrevLedger(mPrevLedgerHash); if (proposal->checkSign()) { - cLog(lsINFO) << "Applying deferred proposal"; + cLog(lsINFO) << "Applying stored proposal"; peerPosition(proposal); } } diff --git a/src/LedgerConsensus.h b/src/LedgerConsensus.h index ffe8db5d3c..a88aee0792 100644 --- a/src/LedgerConsensus.h +++ b/src/LedgerConsensus.h @@ -113,9 +113,6 @@ protected: // Close time estimates std::map mCloseTimes; - // deferred proposals (node ID -> proposals from that peer) - boost::unordered_map< uint160, std::list > mDeferredProposals; - // nodes that have bowed out of this consensus process boost::unordered_set mDeadNodes; @@ -180,16 +177,12 @@ public: bool haveConsensus(); bool peerPosition(const LedgerProposal::pointer&); - void deferProposal(const LedgerProposal::pointer& proposal, const NewcoinAddress& peerPublic); bool peerHasSet(Peer::ref peer, const uint256& set, ripple::TxSetStatus status); bool peerGaveNodes(Peer::ref peer, const uint256& setHash, const std::list& nodeIDs, const std::list< std::vector >& nodeData); - void swapDefer(boost::unordered_map< uint160, std::list > &n) - { mDeferredProposals.swap(n); } - // test/debug void simulate(); }; diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 931364054f..9698bbbc57 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -57,7 +57,7 @@ protected: boost::asio::deadline_timer mNetTimer; boost::shared_ptr mConsensus; boost::unordered_map > mDeferredProposals; + std::list > mStoredProposals; LedgerMaster* mLedgerMaster; LedgerAcquire::pointer mAcquiringLedger; @@ -193,6 +193,9 @@ public: void setLastCloseTime(uint32 t) { mLastCloseTime = t; } Json::Value getServerInfo(); uint32 acceptLedger(); + boost::unordered_map >& peekStoredProposals() { return mStoredProposals; } + void storeProposal(const LedgerProposal::pointer& proposal, const NewcoinAddress& peerPublic); // client information retrieval functions std::vector< std::pair > From c05e42ccb7bc0490bcd854b2cc5696dac2520041 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 05:28:41 -0700 Subject: [PATCH 07/27] Change "deferred proposal" to "stored". Complete move to NetworkOPs. --- src/NetworkOPs.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index af155ae692..883ca81538 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -661,7 +661,6 @@ int NetworkOPs::beginConsensus(const uint256& networkClosed, Ledger::pointer clo prevLedger->setImmutable(); mConsensus = boost::make_shared( networkClosed, prevLedger, mLedgerMaster->getCurrentLedger()->getCloseTimeNC()); - mConsensus->swapDefer(mDeferredProposals); cLog(lsDEBUG) << "Initiating consensus engine"; return mConsensus->startup(); @@ -731,7 +730,7 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, cons } if (prevLedger == mConsensus->getLCL()) return mConsensus->peerPosition(proposal); - mConsensus->deferProposal(proposal, nodePublic); + storeProposal(proposal, nodePublic); return false; } @@ -741,7 +740,7 @@ bool NetworkOPs::recvPropose(uint32 proposeSeq, const uint256& proposeHash, cons { // Note that if the LCL is different, the signature check will fail cLog(lsWARNING) << "Ledger proposal fails signature check"; proposal->setSignature(signature); - mConsensus->deferProposal(proposal, nodePublic); + storeProposal(proposal, nodePublic); return false; } return mConsensus->peerPosition(proposal); @@ -788,7 +787,6 @@ void NetworkOPs::endConsensus(bool correctLCL) cLog(lsTRACE) << "Killing obsolete peer status"; it->cycleStatus(); } - mConsensus->swapDefer(mDeferredProposals); mConsensus = boost::shared_ptr(); } @@ -1247,6 +1245,14 @@ uint32 NetworkOPs::acceptLedger() return mLedgerMaster->getCurrentLedger()->getLedgerSeq(); } +void NetworkOPs::storeProposal(const LedgerProposal::pointer& proposal, const NewcoinAddress& peerPublic) +{ + std::list& props = mStoredProposals[peerPublic.getNodeID()]; + if (props.size() >= (mLastCloseProposers + 10)) + props.pop_front(); + props.push_back(proposal); +} + #if 0 void NetworkOPs::subAccountChanges(InfoSub* ispListener, const uint256 uLedgerHash) { From 81e30be00102bf80b23774b393daceca717e6489 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 06:09:26 -0700 Subject: [PATCH 08/27] Include whether we're waiting for a network ledger in the server_info output. --- src/NetworkOPs.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/NetworkOPs.cpp b/src/NetworkOPs.cpp index 883ca81538..477cca2275 100644 --- a/src/NetworkOPs.cpp +++ b/src/NetworkOPs.cpp @@ -875,8 +875,13 @@ Json::Value NetworkOPs::getServerInfo() default: info["serverState"] = "unknown"; } - if (!theConfig.VALIDATION_SEED.isValid()) info["serverState"] = "none"; - else info["validationPKey"] = NewcoinAddress::createNodePublic(theConfig.VALIDATION_SEED).humanNodePublic(); + if (!theConfig.VALIDATION_SEED.isValid()) + info["serverState"] = "none"; + else + info["validationPKey"] = NewcoinAddress::createNodePublic(theConfig.VALIDATION_SEED).humanNodePublic(); + + if (mNeedNetworkLedger) + info["networkLedger"] = "waiting"; Json::Value lastClose = Json::objectValue; lastClose["proposers"] = theApp->getOPs().getPreviousProposers(); From 21c4c22eda5e1002bfca0351c5968f7568f8f698 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 06:09:43 -0700 Subject: [PATCH 09/27] Add functions to check and clear the "need network ledger" flag. --- src/NetworkOPs.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/NetworkOPs.h b/src/NetworkOPs.h index 9698bbbc57..cd2e8bede0 100644 --- a/src/NetworkOPs.h +++ b/src/NetworkOPs.h @@ -186,6 +186,8 @@ public: void setStateTimer(); void newLCL(int proposers, int convergeTime, const uint256& ledgerHash); void needNetworkLedger() { mNeedNetworkLedger = true; } + void clearNeedNetworkLedger() { mNeedNetworkLedger = false; } + bool isNeedNetworkLedger() { return mNeedNetworkLedger; } void consensusViewChange(); int getPreviousProposers() { return mLastCloseProposers; } int getPreviousConvergeTime() { return mLastCloseConvergeTime; } From 83cfc7a3df848627c4ff52939d8174111beaf2ea Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 06:10:13 -0700 Subject: [PATCH 10/27] Make sure we clear the 'need network ledger' flag if we jump ledgers during consensus. --- src/LedgerConsensus.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index bb56dcde9c..6f4d0aac0e 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -359,6 +359,7 @@ void LedgerConsensus::handleLCL(const uint256& lclHash) cLog(lsINFO) << "Acquired the consensus ledger " << mPrevLedgerHash; mHaveCorrectLCL = true; mAcquiringLedger = LedgerAcquire::pointer(); + theApp->getOPs().clearNeedNetworkLedger(); mCloseResolution = ContinuousLedgerTiming::getNextLedgerTimeResolution( mPreviousLedger->getCloseResolution(), mPreviousLedger->getCloseAgree(), mPreviousLedger->getLedgerSeq() + 1); From 599ebe3c6111ed4c1c746b1fd857ed9e842c5704 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 06:43:34 -0700 Subject: [PATCH 11/27] Write ledger dirty SHA nodes before ledger header. --- src/Ledger.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index b21958e58c..7ee7e2689d 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -363,13 +363,9 @@ void Ledger::saveAcceptedLedger() { ScopedLock sl(theApp->getLedgerDB()->getDBLock()); + if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % mLedgerSeq))) theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % mLedgerSeq)); - theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % - getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() % - boost::lexical_cast(mTotCoins) % mCloseTime % mParentCloseTime % - mCloseResolution % mCloseFlags % - mAccountHash.GetHex() % mTransHash.GetHex())); // write out dirty nodes int fc; @@ -379,6 +375,12 @@ void Ledger::saveAcceptedLedger() { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } disarmDirty(); + theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % + getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() % + boost::lexical_cast(mTotCoins) % mCloseTime % mParentCloseTime % + mCloseResolution % mCloseFlags % + mAccountHash.GetHex() % mTransHash.GetHex())); + SHAMap& txSet = *peekTransactionMap(); Database *db = theApp->getTxnDB()->getDB(); ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); From afe016415e894529b6e2c63198b87da6b85043f8 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 06:53:06 -0700 Subject: [PATCH 12/27] Ensure all dirty SHA nodes are fully written before writing the ledger header. To avoid a performance penalty, write accountstate nodes while we're waiting. --- src/HashedObject.cpp | 12 ++++++++++-- src/HashedObject.h | 8 +++++++- src/Ledger.cpp | 13 +++++++------ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 837a518a66..c1df0b3fad 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -36,7 +36,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, if (!mCache.canonicalize(hash, object)) { // cLog(lsTRACE) << "Queuing write for " << hash; - boost::recursive_mutex::scoped_lock sl(mWriteMutex); + boost::mutex::scoped_lock sl(mWriteMutex); mWriteSet.push_back(object); if (!mWritePending) { @@ -50,6 +50,13 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, return true; } +void HashedObjectStore::waitWrite() +{ + boost::unique_lock sl(mWriteMutex); + while (mWritePending) + mWriteCondition.wait(sl); +} + void HashedObjectStore::bulkWrite() { std::vector< boost::shared_ptr > set; @@ -59,11 +66,12 @@ void HashedObjectStore::bulkWrite() set.reserve(128); { - boost::recursive_mutex::scoped_lock sl(mWriteMutex); + boost::unique_lock sl(mWriteMutex); mWriteSet.swap(set); if (set.empty()) { mWritePending = false; + mWriteCondition.notify_all(); return; } } diff --git a/src/HashedObject.h b/src/HashedObject.h index f403d5042a..5178af0896 100644 --- a/src/HashedObject.h +++ b/src/HashedObject.h @@ -3,6 +3,9 @@ #include +#include +#include + #include "types.h" #include "uint256.h" #include "ScopedLock.h" @@ -41,7 +44,9 @@ class HashedObjectStore protected: TaggedCache mCache; - boost::recursive_mutex mWriteMutex; + boost::mutex mWriteMutex; + boost::condition_variable mWriteCondition; + std::vector< boost::shared_ptr > mWriteSet; bool mWritePending; @@ -55,6 +60,7 @@ public: HashedObject::pointer retrieve(const uint256& hash); void bulkWrite(); + void waitWrite(); }; #endif diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 7ee7e2689d..ceabf60489 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -375,12 +375,6 @@ void Ledger::saveAcceptedLedger() { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } disarmDirty(); - theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % - getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() % - boost::lexical_cast(mTotCoins) % mCloseTime % mParentCloseTime % - mCloseResolution % mCloseFlags % - mAccountHash.GetHex() % mTransHash.GetHex())); - SHAMap& txSet = *peekTransactionMap(); Database *db = theApp->getTxnDB()->getDB(); ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); @@ -437,6 +431,13 @@ void Ledger::saveAcceptedLedger() } } db->executeSQL("COMMIT TRANSACTION;"); + + theApp->getHashedObjectStore().waitWrite(); // wait until all nodes are written + theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % + getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() % + boost::lexical_cast(mTotCoins) % mCloseTime % mParentCloseTime % + mCloseResolution % mCloseFlags % + mAccountHash.GetHex() % mTransHash.GetHex())); } theApp->getOPs().pubLedger(shared_from_this()); From 09f3b5db543f6f45678713527776944edb18c6aa Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 11:16:07 -0700 Subject: [PATCH 13/27] Rework how dirty SHAMap nodes are flushed to disk. The existing code had a bug that could cause nodes to be modified before they were written to disk. --- src/Ledger.cpp | 16 +--------------- src/Ledger.h | 4 ---- src/LedgerConsensus.cpp | 17 ++++++++++++++--- src/SHAMap.h | 9 +++++---- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index ceabf60489..00f8864fc8 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -35,8 +35,7 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(s mAccountStateMap->armDirty(); writeBack(lepCREATE, startAccount->getSLE()); - mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq); - mAccountStateMap->disarmDirty(); + SHAMap::flushDirty(*mAccountStateMap->disarmDirty(), 256, hotACCOUNT_NODE, mLedgerSeq); } Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash, @@ -329,11 +328,6 @@ bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, Tran return true; } -bool Ledger::unitTest() -{ - return true; -} - uint256 Ledger::getHash() { if (!mValidHash) @@ -367,14 +361,6 @@ void Ledger::saveAcceptedLedger() if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % mLedgerSeq))) theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % mLedgerSeq)); - // write out dirty nodes - int fc; - while ((fc = mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, mLedgerSeq)) > 0) - { cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; } - while ((fc = mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq)) > 0) - { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } - disarmDirty(); - SHAMap& txSet = *peekTransactionMap(); Database *db = theApp->getTxnDB()->getDB(); ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); diff --git a/src/Ledger.h b/src/Ledger.h index 0c201d12fe..aa9cc9f142 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -109,8 +109,6 @@ public: bool isClosed() { return mClosed; } bool isAccepted() { return mAccepted; } bool isImmutable() { return mImmutable; } - void armDirty() { mTransactionMap->armDirty(); mAccountStateMap->armDirty(); } - void disarmDirty() { mTransactionMap->disarmDirty(); mAccountStateMap->disarmDirty(); } // ledger signature operations void addRaw(Serializer &s) const; @@ -283,8 +281,6 @@ public: bool walkLedger(); bool assertSane(); - - static bool unitTest(); }; inline LedgerStateParms operator|(const LedgerStateParms& l1, const LedgerStateParms& l2) diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index 6f4d0aac0e..f241934ed9 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -1098,12 +1098,23 @@ void LedgerConsensus::accept(SHAMap::ref set) cLog(lsINFO) << "CNF mode " << theApp->getOPs().getOperatingMode() << ", oldLCL " << mPrevLedgerHash; } - Ledger::pointer newLCL = boost::make_shared(false, boost::ref(*mPreviousLedger)); - newLCL->armDirty(); - CanonicalTXSet failedTransactions(set->getHash()); + + Ledger::pointer newLCL = boost::make_shared(false, boost::ref(*mPreviousLedger)); + + newLCL->peekTransactionMap()->armDirty(); + newLCL->peekAccountStateMap()->armDirty(); applyTransactions(set, newLCL, newLCL, failedTransactions, false); newLCL->setClosed(); + boost::shared_ptr acctNodes = newLCL->peekAccountStateMap()->disarmDirty(); + boost::shared_ptr txnNodes = newLCL->peekTransactionMap()->disarmDirty(); + + // write out dirty nodes (temporarily done here) Most come before setAccepted + int fc; + while ((fc = SHAMap::flushDirty(*acctNodes, 256, hotACCOUNT_NODE, newLCL->getLedgerSeq())) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } + while ((fc = SHAMap::flushDirty(*txnNodes, 256, hotTRANSACTION_NODE, newLCL->getLedgerSeq())) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; } bool closeTimeCorrect = true; if (closeTime == 0) diff --git a/src/SHAMap.h b/src/SHAMap.h index 6d579a918d..f8afce826e 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -283,13 +283,14 @@ public: typedef const boost::shared_ptr& ref; typedef std::map > SHAMapDiff; + typedef boost::unordered_map SHADirtyMap; private: uint32 mSeq; mutable boost::recursive_mutex mLock; boost::unordered_map mTNByID; - boost::shared_ptr< boost::unordered_map > mDirtyNodes; + boost::shared_ptr mDirtyNodes; SHAMapTreeNode::pointer root; @@ -384,9 +385,9 @@ public: // return value: true=successfully completed, false=too different bool compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount); - void armDirty(); - int flushDirty(int maxNodes, HashedObjectType t, uint32 seq); - void disarmDirty(); + int armDirty(); + static int flushDirty(SHADirtyMap& dirtyMap, int maxNodes, HashedObjectType t, uint32 seq); + boost::shared_ptr disarmDirty(); void setSeq(uint32 seq) { mSeq = seq; } uint32 getSeq() { return mSeq; } From 2ebf8a796847394bdaa2dc4bdd5f5f00d658362e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 11:47:01 -0700 Subject: [PATCH 14/27] Cleanup. --- src/LedgerHistory.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index 2c1563d639..75519538a2 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -99,7 +99,9 @@ Ledger::pointer LedgerHistory::canonicalizeLedger(Ledger::pointer ledger, bool s // save input ledger in map if not in map, otherwise return corresponding map ledger boost::recursive_mutex::scoped_lock sl(mLedgersByHash.peekMutex()); mLedgersByHash.canonicalize(h, ledger); - if (ledger->isAccepted()) mLedgersByIndex[ledger->getLedgerSeq()] = ledger; + if (ledger->isAccepted()) + mLedgersByIndex[ledger->getLedgerSeq()] = ledger; return ledger; } + // vim:ts=4 From 68b8360cdac532ea57f150242eb1a6129538b7d4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 11:47:42 -0700 Subject: [PATCH 15/27] Hashes for transaction nodes with metadata were miscomputed. This may cause broken nodes not to stay synched with working nodes. --- src/SHAMapNodes.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 6c02b79a3b..1d7f696231 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -301,6 +301,8 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vectorpeekData()); + Serializer s(mItem->peekSerializer().getDataLength() + (256 + 32) / 8); + s.add32(sHP_TransactionNode); + s.addRaw(mItem->peekData()); + s.add256(mItem->getTag()); + nh = s.getSHA512Half(); } else assert(false); From 6f23c44152635a79893d4722c303fc78d70e84fe Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 11:48:49 -0700 Subject: [PATCH 16/27] Fix a few cases where SHAMap nodes don't get correctly written. When we fetch a node from the database, we artificially mark it "old" so modifications are saved. New nodes are added to the dirty node tray if it is armed. --- src/SHAMap.cpp | 76 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 26 deletions(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 890075215e..a2d9b56353 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -48,7 +48,7 @@ SHAMap::SHAMap(SHAMapType t, uint32 seq) : mSeq(seq), mState(smsModifying), mTyp mTNByID[*root] = root; } -SHAMap::SHAMap(SHAMapType t, const uint256& hash) : mSeq(0), mState(smsSynching), mType(t) +SHAMap::SHAMap(SHAMapType t, const uint256& hash) : mSeq(1), mState(smsSynching), mType(t) { // FIXME: Need to acquire root node root = boost::make_shared(mSeq, SHAMapNode(0, uint256())); root->makeInner(); @@ -138,7 +138,8 @@ void SHAMap::dirtyUp(std::stack& stack, const uint256& SHAMapTreeNode::pointer SHAMap::checkCacheNode(const SHAMapNode& iNode) { boost::unordered_map::iterator it = mTNByID.find(iNode); - if (it == mTNByID.end()) return SHAMapTreeNode::pointer(); + if (it == mTNByID.end()) + return SHAMapTreeNode::pointer(); return it->second; } @@ -209,6 +210,7 @@ SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& id, const uint256& has node = fetchNodeExternal(id, hash); if (!mTNByID.insert(std::make_pair(id, node)).second) assert(false); + trackNewNode(node); return node; } @@ -221,6 +223,7 @@ SHAMapTreeNode* SHAMap::getNodePointer(const SHAMapNode& id, const uint256& hash SHAMapTreeNode::pointer node = fetchNodeExternal(id, hash); if (!mTNByID.insert(std::make_pair(id, node)).second) assert(false); + trackNewNode(node); return node.get(); } @@ -230,16 +233,25 @@ void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify) assert(node->getSeq() <= mSeq); if (node && modify && (node->getSeq() != mSeq)) { // have a CoW - node = boost::make_shared(*node, mSeq); - if (mDirtyNodes) - (*mDirtyNodes)[*node] = node; + assert(node->getSeq() < mSeq); + + node = boost::make_shared(*node, mSeq); // here's to the new node, same as the old node assert(node->isValid()); + mTNByID[*node] = node; if (node->isRoot()) root = node; + if (mDirtyNodes) + (*mDirtyNodes)[*node] = node; } } +void SHAMap::trackNewNode(SHAMapTreeNode::pointer& node) +{ + if (mDirtyNodes) + (*mDirtyNodes)[*node] = node; +} + SHAMapItem::SHAMapItem(const uint256& tag, const std::vector& data) : mTag(tag), mData(data) { ; } @@ -594,6 +606,7 @@ bool SHAMap::addGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta) assert(false); throw std::runtime_error("invalid inner node"); } + trackNewNode(newNode); node->setChildHash(branch, newNode->getNodeHash()); } else @@ -622,6 +635,7 @@ bool SHAMap::addGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta) assert(false); stack.push(node); node = newNode; + trackNewNode(node); } // we can add the two leaf nodes here @@ -632,12 +646,14 @@ bool SHAMap::addGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta) if (!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) assert(false); node->setChildHash(b1, newNode->getNodeHash()); // OPTIMIZEME hash op not needed + trackNewNode(newNode); newNode = boost::make_shared(node->getChildNodeID(b2), otherItem, type, mSeq); assert(newNode->isValid() && newNode->isLeaf()); if (!mTNByID.insert(std::make_pair(SHAMapNode(*newNode), newNode)).second) assert(false); node->setChildHash(b2, newNode->getNodeHash()); + trackNewNode(newNode); } dirtyUp(stack, tag, node->getNodeHash()); @@ -703,8 +719,19 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui try { - SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq, snfPREFIX); - assert((ret->getNodeHash() == hash) && (id == *ret)); + SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq - 1, snfPREFIX); +#ifdef DEBUG + if (id != *ret) + { + Log(lsFATAL) << "id:" << id << ", got:" << *ret; + assert(false); + } + if (ret->getNodeHash() != hash) + { + Log(lsFATAL) << "Hashes don't match"; + assert(false); + } +#endif return ret; } catch (...) @@ -730,40 +757,37 @@ void SHAMap::fetchRoot(const uint256& hash) assert(root->getNodeHash() == hash); } -void SHAMap::armDirty() +int SHAMap::armDirty() { // begin saving dirty nodes - ++mSeq; mDirtyNodes = boost::make_shared< boost::unordered_map >(); + return ++mSeq; } -int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) +int SHAMap::flushDirty(SHADirtyMap& map, int maxNodes, HashedObjectType t, uint32 seq) { int flushed = 0; Serializer s; - if (mDirtyNodes) + for(SHADirtyMap::iterator it = map.begin(); it != map.end(); it = map.erase(it)) { - boost::unordered_map& dirtyNodes = *mDirtyNodes; - boost::unordered_map::iterator it = dirtyNodes.begin(); - while (it != dirtyNodes.end()) - { -// tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first; -// tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first; - s.erase(); - it->second->addRaw(s, snfPREFIX); - theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half()); - if (flushed++ >= maxNodes) - return flushed; - it = dirtyNodes.erase(it); - } +// tLog(t == hotTRANSACTION_NODE, lsDEBUG) << "TX node write " << it->first; +// tLog(t == hotACCOUNT_NODE, lsDEBUG) << "STATE node write " << it->first; + s.erase(); + it->second->addRaw(s, snfPREFIX); + assert(s.getSHA512Half() == it->second->getNodeHash()); + theApp->getHashedObjectStore().store(t, seq, s.peekData(), it->second->getNodeHash()); + if (flushed++ >= maxNodes) + return flushed; } return flushed; } -void SHAMap::disarmDirty() +boost::shared_ptr SHAMap::disarmDirty() { // stop saving dirty nodes - mDirtyNodes = boost::shared_ptr< boost::unordered_map >(); + boost::shared_ptr ret; + ret.swap(mDirtyNodes); + return ret; } SHAMapTreeNode::pointer SHAMap::getNode(const SHAMapNode& nodeID) From 2de864d23b25659b2a28b368f5ef2db5506cd6c0 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 11:49:34 -0700 Subject: [PATCH 17/27] Updates. --- src/SHAMap.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/SHAMap.h b/src/SHAMap.h index f8afce826e..a30c1f8808 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -306,6 +306,7 @@ protected: SHAMapTreeNode* walkToPointer(const uint256& id); SHAMapTreeNode::pointer checkCacheNode(const SHAMapNode&); void returnNode(SHAMapTreeNode::pointer&, bool modify); + void trackNewNode(SHAMapTreeNode::pointer&); SHAMapTreeNode::pointer getNode(const SHAMapNode& id); SHAMapTreeNode::pointer getNode(const SHAMapNode& id, const uint256& hash, bool modify); @@ -322,7 +323,7 @@ protected: public: // build new map - SHAMap(SHAMapType t, uint32 seq = 0); + SHAMap(SHAMapType t, uint32 seq = 1); SHAMap(SHAMapType t, const uint256& hash); ~SHAMap() { mState = smsInvalid; } From cf32d900de61c94132581f3520275f7190ce3351 Mon Sep 17 00:00:00 2001 From: jed Date: Tue, 16 Oct 2012 12:40:07 -0700 Subject: [PATCH 18/27] windows --- newcoin.vcxproj | 4 ++-- newcoin.vcxproj.filters | 6 +++--- src/RPCServer.cpp | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/newcoin.vcxproj b/newcoin.vcxproj index d83a20bd48..67586127c6 100644 --- a/newcoin.vcxproj +++ b/newcoin.vcxproj @@ -49,7 +49,7 @@ Level3 Disabled BOOST_TEST_ALTERNATIVE_INIT_API;BOOST_TEST_NO_MAIN;_CRT_SECURE_NO_WARNINGS;_WIN32_WINNT=0x0501;WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - ..\OpenSSL\include;..\boost_1_47_0;..\protobuf-2.4.1\src\ + .\obj;..\OpenSSL\include;..\boost_1_47_0;..\protobuf-2.4.1\src\ ProgramDatabase @@ -93,7 +93,7 @@ - + diff --git a/newcoin.vcxproj.filters b/newcoin.vcxproj.filters index 2c3a38dac3..7941c68ec6 100644 --- a/newcoin.vcxproj.filters +++ b/newcoin.vcxproj.filters @@ -201,9 +201,6 @@ Source Files\database - - Source Files - Source Files @@ -303,6 +300,9 @@ Source Files + + Source Files + diff --git a/src/RPCServer.cpp b/src/RPCServer.cpp index 1c88bb2082..07d8376cb8 100644 --- a/src/RPCServer.cpp +++ b/src/RPCServer.cpp @@ -133,7 +133,7 @@ void RPCServer::handle_read(const boost::system::error_code& e, } else if (!result) { // bad request - std::cout << "bad request" << std::endl; + std::cout << "bad request: " << mIncomingRequest.mBody < Date: Tue, 16 Oct 2012 13:07:22 -0700 Subject: [PATCH 19/27] Some additional helper functions. --- src/SerializedObject.cpp | 13 +++++++++++++ src/SerializedObject.h | 3 +++ 2 files changed, 16 insertions(+) diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index c8d243952e..a7ce22ba43 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -396,6 +396,19 @@ bool STObject::isFieldPresent(SField::ref field) const return peekAtIndex(index).getSType() != STI_NOTPRESENT; } +STObject& STObject::peekFieldObject(SField::ref field) +{ + SerializedType* rf = getPField(field, true); + if (!rf) + throw std::runtime_error("Field not found"); + if (rf->getSType() == STI_NOTPRESENT) + rf = makeFieldPresent(field); + STObject* cf = dynamic_cast(rf); + if (!cf) + throw std::runtime_error("Wrong field type"); + return *cf; +} + bool STObject::setFlag(uint32 f) { STUInt32* t = dynamic_cast(getPField(sfFlags, true)); diff --git a/src/SerializedObject.h b/src/SerializedObject.h index f2c389f09d..69683ed497 100644 --- a/src/SerializedObject.h +++ b/src/SerializedObject.h @@ -132,6 +132,8 @@ public: void setFieldPathSet(SField::ref field, const STPathSet&); void setFieldV256(SField::ref field, const STVector256& v); + STObject& peekFieldObject(SField::ref field); + bool isFieldPresent(SField::ref field) const; SerializedType* makeFieldPresent(SField::ref field); void makeFieldAbsent(SField::ref field); @@ -199,6 +201,7 @@ public: void pop_back() { value.pop_back(); } bool empty() const { return value.empty(); } void clear() { value.clear(); } + void swap(STArray& a) { value.swap(a.value); } virtual std::string getFullText() const; virtual std::string getText() const; From 1f89317c07efb75b17748d8f582a26ca8090a73e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 13:55:34 -0700 Subject: [PATCH 20/27] Fix JSON for arrays. --- src/SerializedObject.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index a7ce22ba43..0ba32127b4 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -831,8 +831,20 @@ std::string STArray::getText() const Json::Value STArray::getJson(int p) const { Json::Value v = Json::arrayValue; - BOOST_FOREACH(const STObject& o, value) - v.append(o.getJson(p)); + int index = 1; + BOOST_FOREACH(const STObject& object, value) + { + if (object.getSType() != STI_NOTPRESENT) + { + Json::Value inner = Json::objectValue; + if (!object.getFName().hasName()) + inner[lexical_cast_i(index)] = object.getJson(p); + else + inner[object.getName()] = object.getJson(p); + v.append(inner); + index++; + } + } return v; } From 9e8a7d4ced47e35da4875f8d67ea2428eebd5fe0 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 13:55:57 -0700 Subject: [PATCH 21/27] Remove all the old txn meta code. It's not obsolete --- src/TransactionMeta.cpp | 347 +++++++--------------------------------- src/TransactionMeta.h | 172 ++------------------ 2 files changed, 63 insertions(+), 456 deletions(-) diff --git a/src/TransactionMeta.cpp b/src/TransactionMeta.cpp index d1372ebdc7..33840cc108 100644 --- a/src/TransactionMeta.cpp +++ b/src/TransactionMeta.cpp @@ -6,314 +6,50 @@ #include #include -bool TransactionMetaNodeEntry::operator<(const TransactionMetaNodeEntry& e) const -{ - if (mType < e.mType) return true; - if (mType > e.mType) return false; - return compare(e) < 0; -} - -bool TransactionMetaNodeEntry::operator<=(const TransactionMetaNodeEntry& e) const -{ - if (mType < e.mType) return true; - if (mType > e.mType) return false; - return compare(e) <= 0; -} - -bool TransactionMetaNodeEntry::operator>(const TransactionMetaNodeEntry& e) const -{ - if (mType > e.mType) return true; - if (mType < e.mType) return false; - return compare(e) > 0; -} - -bool TransactionMetaNodeEntry::operator>=(const TransactionMetaNodeEntry& e) const -{ - if (mType > e.mType) return true; - if (mType < e.mType) return false; - return compare(e) >= 0; -} - -TMNEThread::TMNEThread(SerializerIterator& sit) : TransactionMetaNodeEntry(TMSThread) -{ - mPrevTxID = sit.get256(); - mPrevLgrSeq = sit.get32(); -} - -void TMNEThread::addRaw(Serializer& sit) const -{ - sit.add8(mType); - sit.add256(mPrevTxID); - sit.add32(mPrevLgrSeq); -} - -int TMNEThread::compare(const TransactionMetaNodeEntry&) const -{ - assert(false); // should never be two entries for the same node (as of now) - return 0; -} - -Json::Value TMNEThread::getJson(int) const -{ - Json::Value inner(Json::objectValue); - inner["prev_transaction"] = mPrevTxID.GetHex(); - inner["prev_ledger_seq"] = mPrevLgrSeq; - - Json::Value outer(Json::objectValue); - outer["thread"] = inner; - return outer; -} - -TMNEAmount::TMNEAmount(int type, SerializerIterator& sit) : TransactionMetaNodeEntry(type) -{ - mAmount = *dynamic_cast(STAmount::deserialize(sit, sfAmount).get()); // Ouch -} - -void TMNEAmount::addRaw(Serializer& s) const -{ - s.add8(mType); - mAmount.add(s); -} - -Json::Value TMNEAmount::getJson(int v) const -{ - Json::Value outer(Json::objectValue); - switch (mType) - { - case TMSPrevBalance: outer["prev_balance"] = mAmount.getJson(v); break; - case TMSFinalBalance: outer["final_balance"] = mAmount.getJson(v); break; - case TMSPrevTakerPays: outer["prev_taker_pays"] = mAmount.getJson(v); break; - case TMSPrevTakerGets: outer["prev_taker_gets"] = mAmount.getJson(v); break; - case TMSFinalTakerPays: outer["final_taker_pays"] = mAmount.getJson(v); break; - case TMSFinalTakerGets: outer["final_taker_gets"] = mAmount.getJson(v); break; - default: assert(false); - } - return outer; -} - -int TMNEAmount::compare(const TransactionMetaNodeEntry& e) const -{ - assert(false); // can't be two changed amounts of same type - return 0; -} - -TMNEAccount::TMNEAccount(int type, SerializerIterator& sit) - : TransactionMetaNodeEntry(type), mAccount(STAccount(sit.getVL()).getValueNCA()) -{ ; } - -void TMNEAccount::addRaw(Serializer& sit) const -{ - sit.add8(mType); - - STAccount sta; - sta.setValueNCA(mAccount); - sta.add(sit); -} - -Json::Value TMNEAccount::getJson(int) const -{ - Json::Value outer(Json::objectValue); - switch (mType) - { - case TMSPrevAccount: outer["prev_account"] = mAccount.humanAccountID(); break; - case TMSLowID: outer["lowID"] = mAccount.humanAccountID(); break; - case TMSHighID: outer["highID"] = mAccount.humanAccountID(); break; - default: assert(false); - } - return outer; -} - -int TMNEAccount::compare(const TransactionMetaNodeEntry&) const -{ - assert(false); // Can't be two modified accounts of same type for same node - return 0; -} - -TransactionMetaNode::TransactionMetaNode(int type, const uint256& node, SerializerIterator& sit) - : mType(type), mNode(node) -{ - while (1) - { - int nType = sit.get8(); - switch (nType) - { - case TMSEndOfNode: - return; - - case TMSThread: - mEntries.push_back(new TMNEThread(sit)); - break; - - // Nodes that contain an amount - case TMSPrevBalance: - case TMSPrevTakerPays: - case TMSPrevTakerGets: - case TMSFinalTakerPays: - case TMSFinalTakerGets: - mEntries.push_back(new TMNEAmount(nType, sit)); - - case TMSPrevAccount: - mEntries.push_back(new TMNEAccount(nType, sit)); - } - } -} - -void TransactionMetaNode::addRaw(Serializer& s) -{ - s.add8(mType); - s.add256(mNode); - mEntries.sort(); - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - it.addRaw(s); - s.add8(TMSEndOfNode); -} - -TransactionMetaNodeEntry* TransactionMetaNode::findEntry(int nodeType) -{ - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - if (it.getType() == nodeType) - return ⁢ - return NULL; -} - -TMNEAmount* TransactionMetaNode::findAmount(int nType) -{ - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - if (it.getType() == nType) - return dynamic_cast(&it); - TMNEAmount* node = new TMNEAmount(nType); - mEntries.push_back(node); - return node; -} - -void TransactionMetaNode::addNode(TransactionMetaNodeEntry* node) -{ - mEntries.push_back(node); -} - -bool TransactionMetaNode::thread(const uint256& prevTx, uint32 prevLgr) -{ - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - if (it.getType() == TMSThread) - { - TMNEThread* a = dynamic_cast(&it); - assert(a && (a->getPrevTxID() == prevTx) && (a->getPrevLgr() == prevLgr)); - return false; - } - addNode(new TMNEThread(prevTx, prevLgr)); - return true; -} - -bool TransactionMetaNode::addAmount(int nodeType, const STAmount& amount) -{ - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - if (it.getType() == nodeType) - { - TMNEAmount* a = dynamic_cast(&it); - assert(a && (a->getAmount() == amount)); - return false; - } - addNode(new TMNEAmount(nodeType, amount)); - return true; -} - -bool TransactionMetaNode::addAccount(int nodeType, const NewcoinAddress& account) -{ - BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) - if (it.getType() == nodeType) - { - TMNEAccount* a = dynamic_cast(&it); - assert(a && (a->getAccount() == account)); - return false; - } - addNode(new TMNEAccount(nodeType, account)); - return true; -} - -Json::Value TransactionMetaNode::getJson(int v) const -{ - Json::Value ret = Json::objectValue; - - switch (mType) - { - case TMNCreatedNode: ret["action"] = "create"; break; - case TMNDeletedNode: ret["action"] = "delete"; break; - case TMNModifiedNode: ret["action"] = "modify"; break; - default: - assert(false); - } - - ret["node"] = mNode.GetHex(); - - Json::Value e = Json::arrayValue; - BOOST_FOREACH(const TransactionMetaNodeEntry& it, mEntries) - e.append(it.getJson(v)); - ret["entries"] = e; - - return ret; -} - -TransactionMetaSet::TransactionMetaSet(uint32 ledger, const std::vector& vec) : mLedger(ledger) +TransactionMetaSet::TransactionMetaSet(const uint256& txid, uint32 ledger, const std::vector& vec) : + mTransactionID(txid), mLedger(ledger), mNodes(sfTransactionMetaData) { Serializer s(vec); SerializerIterator sit(s); - mTransactionID = sit.get256(); - - int type; - while ((type = sit.get8()) != TMNEndOfMetadata) - { - uint256 node = sit.get256(); - mNodes.insert(std::make_pair(node, TransactionMetaNode(type, node, sit))); - } -} - -void TransactionMetaSet::addRaw(Serializer& s) -{ - s.add256(mTransactionID); - for (std::map::iterator it = mNodes.begin(), end = mNodes.end(); it != end; ++it) - it->second.addRaw(s); - s.add8(TMNEndOfMetadata); -} - -Json::Value TransactionMetaSet::getJson(int v) const -{ - Json::Value ret = Json::objectValue; - - ret["hash"] = mTransactionID.GetHex(); - ret["ledger"] = mLedger; - - Json::Value e = Json::arrayValue; - for (std::map::const_iterator it = mNodes.begin(), end = mNodes.end(); - it != end; ++it) - e.append(it->second.getJson(v)); - ret["nodes_affected"] = e; - - return ret; + std::auto_ptr obj = STArray::deserialize(sit, sfTransactionMetaData); + mNodes = * static_cast(obj.get()); } bool TransactionMetaSet::isNodeAffected(const uint256& node) const { - return mNodes.find(node) != mNodes.end(); + for (STArray::const_iterator it = mNodes.begin(); it != mNodes.end(); ++it) + if (it->getFieldH256(sfLedgerIndex) == node) + return true; + return false; } -TransactionMetaNode& TransactionMetaSet::getAffectedNode(const uint256& node, int type, bool overrideType) +STObject& TransactionMetaSet::getAffectedNode(const uint256& node, SField::ref type, bool overrideType) { - std::map::iterator it = mNodes.find(node); - if (it != mNodes.end()) + for (STArray::iterator it = mNodes.begin(); it != mNodes.end(); ++it) { - if (overrideType) - it->second.setType(type); - return it->second; + if (it->getFieldH256(sfLedgerIndex) == node) + { + if (overrideType) + it->setFName(type); + return *it; + } } - return mNodes.insert(std::make_pair(node, TransactionMetaNode(node, type))).first->second; + + mNodes.push_back(STObject(type)); + STObject& obj = mNodes.back(); + + assert(obj.getFName() == type); + obj.setFieldH256(sfLedgerIndex, node); + + return mNodes.back(); } -const TransactionMetaNode& TransactionMetaSet::peekAffectedNode(const uint256& node) const +const STObject& TransactionMetaSet::peekAffectedNode(const uint256& node) const { - std::map::const_iterator it = mNodes.find(node); - if (it != mNodes.end()) - return it->second; + for (STArray::const_iterator it = mNodes.begin(); it != mNodes.end(); ++it) + if (it->getFieldH256(sfLedgerIndex) == node) + return *it; throw std::runtime_error("Affected node not found"); } @@ -321,7 +57,7 @@ void TransactionMetaSet::init(const uint256& id, uint32 ledger) { mTransactionID = id; mLedger = ledger; - mNodes.clear(); + mNodes = STArray(sfTransactionMetaData); } void TransactionMetaSet::swap(TransactionMetaSet& s) @@ -330,4 +66,29 @@ void TransactionMetaSet::swap(TransactionMetaSet& s) mNodes.swap(s.mNodes); } +bool TransactionMetaSet::thread(STObject& node, const uint256& prevTxID, uint32 prevLgrID) +{ + if (node.getFieldIndex(sfLastTxnID) == -1) + { + assert(node.getFieldIndex(sfLastTxnSeq) == -1); + node.setFieldH256(sfLastTxnID, prevTxID); + node.setFieldU32(sfLastTxnSeq, prevLgrID); + return true; + } + assert(node.getFieldH256(sfLastTxnID) == prevTxID); + assert(node.getFieldU32(sfLastTxnSeq) == prevLgrID); + return false; +} + +static bool compare(const STObject& o1, const STObject& o2) +{ + return o1.getFieldH256(sfLedgerIndex) < o2.getFieldH256(sfLedgerIndex); +} + +void TransactionMetaSet::addRaw(Serializer& s) +{ + mNodes.sort(compare); + mNodes.add(s); +} + // vim:ts=4 diff --git a/src/TransactionMeta.h b/src/TransactionMeta.h index b329fdb61c..90623b8de7 100644 --- a/src/TransactionMeta.h +++ b/src/TransactionMeta.h @@ -11,164 +11,7 @@ #include "uint256.h" #include "Serializer.h" #include "SerializedTypes.h" - -// master record types -static const int TMNEndOfMetadata = 0x00; -static const int TMNCreatedNode = 0x10; // This transaction created this node -static const int TMNDeletedNode = 0x11; -static const int TMNModifiedNode = 0x12; - -// sub record types - special -static const int TMSEndOfNode = 0x00; -static const int TMSThread = 0x01; // Holds previous TxID and LgrSeq for threading - -// sub record types - containing an amount -static const int TMSPrevBalance = 0x11; // Balances prior to the transaction -static const int TMSFinalBalance = 0x12; // deleted with non-zero balance -static const int TMSPrevTakerPays = 0x13; -static const int TMSPrevTakerGets = 0x14; -static const int TMSFinalTakerPays = 0x15; // Balances at node deletion time -static const int TMSFinalTakerGets = 0x16; - -// sub record types - containing an account (for example, for when a nickname is transferred) -static const int TMSPrevAccount = 0x20; -static const int TMSLowID = 0x21; -static const int TMSHighID = 0x22; - - -class TransactionMetaNodeEntry -{ // a way that a transaction has affected a node -public: - typedef boost::shared_ptr pointer; - -protected: - int mType; - -public: - TransactionMetaNodeEntry(int type) : mType(type) { ; } - virtual ~TransactionMetaNodeEntry() { ; } - - int getType() const { return mType; } - virtual Json::Value getJson(int) const = 0; - virtual void addRaw(Serializer&) const = 0; - - bool operator<(const TransactionMetaNodeEntry&) const; - bool operator<=(const TransactionMetaNodeEntry&) const; - bool operator>(const TransactionMetaNodeEntry&) const; - bool operator>=(const TransactionMetaNodeEntry&) const; - - virtual std::auto_ptr clone() const - { return std::auto_ptr(duplicate()); } - -protected: - virtual int compare(const TransactionMetaNodeEntry&) const = 0; - virtual TransactionMetaNodeEntry* duplicate(void) const = 0; -}; - -class TMNEThread : public TransactionMetaNodeEntry -{ -protected: - uint256 mPrevTxID; - uint32 mPrevLgrSeq; - -public: - TMNEThread() : TransactionMetaNodeEntry(TMSThread), mPrevLgrSeq(0) { ; } - TMNEThread(uint256 prevTx, uint32 prevLgrSeq) - : TransactionMetaNodeEntry(TMSThread), mPrevTxID(prevTx), mPrevLgrSeq(prevLgrSeq) - { ; } - TMNEThread(SerializerIterator&); - - virtual void addRaw(Serializer&) const; - virtual Json::Value getJson(int) const; - - const uint256& getPrevTxID() const { return mPrevTxID; } - uint32 getPrevLgr() const { return mPrevLgrSeq; } - -protected: - virtual TransactionMetaNodeEntry* duplicate(void) const { return new TMNEThread(*this); } - virtual int compare(const TransactionMetaNodeEntry&) const; -}; - -class TMNEAmount : public TransactionMetaNodeEntry -{ // a transaction affected the balance of a node -protected: - STAmount mAmount; - -public: - TMNEAmount(int type) : TransactionMetaNodeEntry(type) { ; } - TMNEAmount(int type, const STAmount &a) : TransactionMetaNodeEntry(type), mAmount(a) { ; } - - TMNEAmount(int type, SerializerIterator&); - virtual void addRaw(Serializer&) const; - - const STAmount& getAmount() const { return mAmount; } - void setAmount(const STAmount& a) { mAmount = a; } - - virtual Json::Value getJson(int) const; - -protected: - virtual TransactionMetaNodeEntry* duplicate(void) const { return new TMNEAmount(*this); } - virtual int compare(const TransactionMetaNodeEntry&) const; -}; - -class TMNEAccount : public TransactionMetaNodeEntry -{ // node was deleted because it was unfunded -protected: - NewcoinAddress mAccount; - -public: - TMNEAccount(int type, const NewcoinAddress& acct) : TransactionMetaNodeEntry(type), mAccount(acct) { ; } - TMNEAccount(int type, SerializerIterator&); - virtual void addRaw(Serializer&) const; - virtual Json::Value getJson(int) const; - - const NewcoinAddress& getAccount() const { return mAccount; } - void setAccount(const NewcoinAddress& a) { mAccount = a; } - -protected: - virtual TransactionMetaNodeEntry* duplicate(void) const { return new TMNEAccount(*this); } - virtual int compare(const TransactionMetaNodeEntry&) const; -}; - -inline TransactionMetaNodeEntry* new_clone(const TransactionMetaNodeEntry& s) { return s.clone().release(); } -inline void delete_clone(const TransactionMetaNodeEntry* s) { boost::checked_delete(s); } - -class TransactionMetaNode -{ // a node that has been affected by a transaction -public: - typedef boost::shared_ptr pointer; - -protected: - int mType; - uint256 mNode; - boost::ptr_vector mEntries; - -public: - TransactionMetaNode(const uint256 &node, int type) : mType(type), mNode(node) { ; } - - const uint256& getNode() const { return mNode; } - const boost::ptr_vector& peekEntries() const { return mEntries; } - - TransactionMetaNodeEntry* findEntry(int nodeType); - void addNode(TransactionMetaNodeEntry*); - - bool operator<(const TransactionMetaNode& n) const { return mNode < n.mNode; } - bool operator<=(const TransactionMetaNode& n) const { return mNode <= n.mNode; } - bool operator>(const TransactionMetaNode& n) const { return mNode > n.mNode; } - bool operator>=(const TransactionMetaNode& n) const { return mNode >= n.mNode; } - - bool thread(const uint256& prevTx, uint32 prevLgr); - - TransactionMetaNode(int type, const uint256& node, SerializerIterator&); - void addRaw(Serializer&); - void setType(int t) { mType = t; } - Json::Value getJson(int) const; - - bool addAmount(int nodeType, const STAmount& amount); - bool addAccount(int nodeType, const NewcoinAddress& account); - TMNEAmount* findAmount(int nodeType); -}; - +#include "SerializedObject.h" class TransactionMetaSet { @@ -178,12 +21,13 @@ public: protected: uint256 mTransactionID; uint32 mLedger; - std::map mNodes; // must be an ordered set + + STArray mNodes; public: TransactionMetaSet() : mLedger(0) { ; } TransactionMetaSet(const uint256& txID, uint32 ledger) : mTransactionID(txID), mLedger(ledger) { ; } - TransactionMetaSet(uint32 ledger, const std::vector&); + TransactionMetaSet(const uint256& txID, uint32 ledger, const std::vector&); void init(const uint256& transactionID, uint32 ledger); void clear() { mNodes.clear(); } @@ -193,11 +37,13 @@ public: uint32 getLgrSeq() { return mLedger; } bool isNodeAffected(const uint256&) const; - TransactionMetaNode& getAffectedNode(const uint256&, int type, bool overrideType); - const TransactionMetaNode& peekAffectedNode(const uint256&) const; + STObject& getAffectedNode(const uint256&, SField::ref type, bool overrideType); + const STObject& peekAffectedNode(const uint256&) const; - Json::Value getJson(int) const; + Json::Value getJson(int p) const { return mNodes.getJson(p); } void addRaw(Serializer&); + + static bool thread(STObject& node, const uint256& prevTxID, uint32 prevLgrID); }; #endif From a9447fdfd13064ce01a2a3a7288dd0e33c4b0237 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 13:56:13 -0700 Subject: [PATCH 22/27] Use new metadata code. --- src/Ledger.cpp | 4 ++-- src/LedgerEntrySet.cpp | 41 +++++++++++++++++++++-------------------- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Ledger.cpp b/src/Ledger.cpp index 00f8864fc8..697e0eb1e2 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -317,7 +317,7 @@ bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, Tran txn = Transaction::sharedTransaction(it.getVL(), true); else it.getVL(); // skip transaction - meta = boost::make_shared(mLedgerSeq, it.getVL()); + meta = boost::make_shared(txID, mLedgerSeq, it.getVL()); } else return false; @@ -554,7 +554,7 @@ Json::Value Ledger::getJson(int options) SerializerIterator tsit(sTxn); SerializedTransaction txn(tsit); - TransactionMetaSet meta(mLedgerSeq, sit.getVL()); + TransactionMetaSet meta(item->getTag(), mLedgerSeq, sit.getVL()); Json::Value txJson = txn.getJson(0); txJson["metaData"] = meta.getJson(0); txns.append(txJson); diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index e804f33c25..64321a271e 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -318,7 +318,8 @@ bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, uint32 prevLgrID; if (!threadTo->thread(mSet.getTxID(), mSet.getLgrSeq(), prevTxID, prevLgrID)) return false; - if (mSet.getAffectedNode(threadTo->getIndex(), TMNModifiedNode, false).thread(prevTxID, prevLgrID)) + if (TransactionMetaSet::thread(mSet.getAffectedNode(threadTo->getIndex(), sfModifiedNode, false), + prevTxID, prevLgrID)) return true; assert(false); return false; @@ -356,7 +357,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) for (std::map::const_iterator it = mEntries.begin(), end = mEntries.end(); it != end; ++it) { - int nType = TMNEndOfMetadata; + SField::ptr type = &sfGeneric; switch (it->second.mAction) { @@ -364,28 +365,28 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) #ifdef META_DEBUG cLog(lsTRACE) << "Modified Node " << it->first; #endif - nType = TMNModifiedNode; + type = &sfModifiedNode; break; case taaDELETE: #ifdef META_DEBUG cLog(lsTRACE) << "Deleted Node " << it->first; #endif - nType = TMNDeletedNode; + type = &sfDeletedNode; break; case taaCREATE: #ifdef META_DEBUG cLog(lsTRACE) << "Created Node " << it->first; #endif - nType = TMNCreatedNode; + type = &sfCreatedNode; break; default: // ignore these break; } - if (nType == TMNEndOfMetadata) + if (type == &sfGeneric) continue; SLE::pointer origNode = mLedger->getSLE(it->first); @@ -394,9 +395,9 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) continue; SLE::pointer curNode = it->second.mEntry; - TransactionMetaNode &metaNode = mSet.getAffectedNode(it->first, nType, true); + STObject &metaNode = mSet.getAffectedNode(it->first, *type, true); - if (nType == TMNDeletedNode) + if (type == &sfDeletedNode) { assert(origNode); threadOwners(origNode, mLedger, newMod); @@ -405,16 +406,16 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) { // node has an amount, covers ripple state nodes STAmount amount = origNode->getFieldAmount(sfAmount); if (amount.isNonZero()) - metaNode.addAmount(TMSPrevBalance, amount); + metaNode.setFieldAmount(sfPreviousBalance, amount); amount = curNode->getFieldAmount(sfAmount); if (amount.isNonZero()) - metaNode.addAmount(TMSFinalBalance, amount); + metaNode.setFieldAmount(sfFinalBalance, amount); if (origNode->getType() == ltRIPPLE_STATE) { - metaNode.addAccount(TMSLowID, + metaNode.setFieldAccount(sfLowID, NewcoinAddress::createAccountID(origNode->getFieldAmount(sfLowLimit).getIssuer())); - metaNode.addAccount(TMSHighID, + metaNode.setFieldAccount(sfHighID, NewcoinAddress::createAccountID(origNode->getFieldAmount(sfHighLimit).getIssuer())); } } @@ -423,44 +424,44 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) { // check for non-zero balances STAmount amount = origNode->getFieldAmount(sfTakerPays); if (amount.isNonZero()) - metaNode.addAmount(TMSFinalTakerPays, amount); + metaNode.setFieldAmount(sfFinalTakerPays, amount); amount = origNode->getFieldAmount(sfTakerGets); if (amount.isNonZero()) - metaNode.addAmount(TMSFinalTakerGets, amount); + metaNode.setFieldAmount(sfFinalTakerGets, amount); } } - if (nType == TMNCreatedNode) // if created, thread to owner(s) + if (type == &sfCreatedNode) // if created, thread to owner(s) { assert(!origNode); threadOwners(curNode, mLedger, newMod); } - if ((nType == TMNCreatedNode) || (nType == TMNModifiedNode)) + if ((type == &sfCreatedNode) || (type == &sfModifiedNode)) { if (curNode->isThreadedType()) // always thread to self threadTx(curNode, mLedger, newMod); } - if (nType == TMNModifiedNode) + if (type == &sfModifiedNode) { assert(origNode); if (origNode->isFieldPresent(sfAmount)) { // node has an amount, covers account root nodes and ripple nodes STAmount amount = origNode->getFieldAmount(sfAmount); if (amount != curNode->getFieldAmount(sfAmount)) - metaNode.addAmount(TMSPrevBalance, amount); + metaNode.setFieldAmount(sfPreviousBalance, amount); } if (origNode->getType() == ltOFFER) { STAmount amount = origNode->getFieldAmount(sfTakerPays); if (amount != curNode->getFieldAmount(sfTakerPays)) - metaNode.addAmount(TMSPrevTakerPays, amount); + metaNode.setFieldAmount(sfPreviousTakerPays, amount); amount = origNode->getFieldAmount(sfTakerGets); if (amount != curNode->getFieldAmount(sfTakerGets)) - metaNode.addAmount(TMSPrevTakerGets, amount); + metaNode.setFieldAmount(sfPreviousTakerGets, amount); } } From d9262e5cdbf811d14a647164ca7c2e6f8794ca27 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 13:56:23 -0700 Subject: [PATCH 23/27] Fix a backwards assert. --- src/SHAMap.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index a2d9b56353..b556a18021 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -774,7 +774,13 @@ int SHAMap::flushDirty(SHADirtyMap& map, int maxNodes, HashedObjectType t, uint3 // tLog(t == hotACCOUNT_NODE, lsDEBUG) << "STATE node write " << it->first; s.erase(); it->second->addRaw(s, snfPREFIX); - assert(s.getSHA512Half() == it->second->getNodeHash()); + if (s.getSHA512Half() != it->second->getNodeHash()) + { + cLog(lsFATAL) << *(it->second); + cLog(lsFATAL) << lexical_cast_i(s.getDataLength()); + cLog(lsFATAL) << s.getSHA512Half() << " != " << it->second->getNodeHash(); + assert(false); + } theApp->getHashedObjectStore().store(t, seq, s.peekData(), it->second->getNodeHash()); if (flushed++ >= maxNodes) return flushed; From cab69318b116308cd1b7f6eddb97f261f3999795 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 13:56:37 -0700 Subject: [PATCH 24/27] Better debug and a data corruption bugfix. --- src/SHAMapNodes.cpp | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 1d7f696231..2eaa0ed806 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -300,7 +300,7 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vector(mHashes), sizeof(mHashes)); } - else if (mType == tnACCOUNT_STATE) - { - Serializer s((256 + 32) / 8 + mItem->peekData().size()); - s.add32(sHP_LeafNode); - mItem->addRaw(s); - s.add256(mItem->getTag()); - nh = s.getSHA512Half(); - } else if (mType == tnTRANSACTION_NM) { nh = Serializer::getPrefixHash(sHP_TransactionID, mItem->peekData()); } + else if (mType == tnACCOUNT_STATE) + { + Serializer s(mItem->peekSerializer().getDataLength() + (256 + 32) / 8); + s.add32(sHP_LeafNode); + s.addRaw(mItem->peekData()); + s.add256(mItem->getTag()); + nh = s.getSHA512Half(); + } else if (mType == tnTRANSACTION_MD) { Serializer s(mItem->peekSerializer().getDataLength() + (256 + 32) / 8); @@ -433,6 +433,7 @@ void SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format) { s.add32(sHP_TransactionNode); mItem->addRaw(s); + s.add256(mItem->getTag()); } else { @@ -511,11 +512,21 @@ std::string SHAMapTreeNode::getString() const } if (isLeaf()) { - ret += ",leaf\n"; + if (mType == tnTRANSACTION_NM) + ret += ",txn\n"; + else if (mType == tnTRANSACTION_MD) + ret += ",txn+md\n"; + else if (mType == tnACCOUNT_STATE) + ret += ",as\n"; + else + ret += ",leaf\n"; + ret += " Tag="; ret += getTag().GetHex(); ret += "\n Hash="; ret += mHash.GetHex(); + ret += "/"; + ret += lexical_cast_i(mItem->peekSerializer().getDataLength()); } return ret; } From 542dc969f510bdeb43f5df335f335fc4891663b2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 14:17:09 -0700 Subject: [PATCH 25/27] Updates. --- src/SerializeProto.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/SerializeProto.h b/src/SerializeProto.h index e3b594db7c..f7d46a50fd 100644 --- a/src/SerializeProto.h +++ b/src/SerializeProto.h @@ -137,13 +137,14 @@ // inner object // OBJECT/1 is reserved for end of object FIELD(TemplateEntry, OBJECT, 1) - FIELD(CreatedNode, OBJECT, 2) - FIELD(DeletedNode, OBJECT, 3) - FIELD(ModifiedNode, OBJECT, 4) + FIELD(TransactionMetaData, OBJECT, 2) + FIELD(CreatedNode, OBJECT, 3) + FIELD(DeletedNode, OBJECT, 4) + FIELD(ModifiedNode, OBJECT, 5) // array of objects // ARRAY/1 is reserved for end of array - FIELD(TransactionMetaData, ARRAY, 1) + FIELD(AffectedNodes, ARRAY, 1) FIELD(SigningAccounts, ARRAY, 2) FIELD(TxnSignatures, ARRAY, 3) FIELD(Signatures, ARRAY, 4) From 0ae65b5fd0cce94d0c837ab3c0d27b16ec724716 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 14:33:51 -0700 Subject: [PATCH 26/27] Some UInt8 support was missing. --- src/SerializedObject.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index 0ba32127b4..6bcae3c067 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -24,6 +24,9 @@ std::auto_ptr STObject::makeDefaultObject(SerializedTypeID id, S case STI_NOTPRESENT: return std::auto_ptr(new SerializedType(name)); + case STI_UINT8: + return std::auto_ptr(new STUInt8(name)); + case STI_UINT16: return std::auto_ptr(new STUInt16(name)); @@ -64,6 +67,8 @@ std::auto_ptr STObject::makeDefaultObject(SerializedTypeID id, S return std::auto_ptr(new STArray(name)); default: + cLog(lsFATAL) << "Object type: " << lexical_cast_i(id); + assert(false); throw std::runtime_error("Unknown object type"); } } @@ -76,6 +81,9 @@ std::auto_ptr STObject::makeDeserializedObject(SerializedTypeID case STI_NOTPRESENT: return SerializedType::deserialize(name); + case STI_UINT8: + return STUInt8::deserialize(sit, name); + case STI_UINT16: return STUInt16::deserialize(sit, name); From 25d888ff48c0cb198f7cee98bdce8d8596885b19 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 16 Oct 2012 14:36:36 -0700 Subject: [PATCH 27/27] Include the transaction result in the metadata. --- src/LedgerEntrySet.cpp | 4 ++-- src/LedgerEntrySet.h | 2 +- src/TransactionEngine.cpp | 2 +- src/TransactionMeta.cpp | 35 +++++++++++++++++++++++++++++------ src/TransactionMeta.h | 14 +++++++++----- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index 64321a271e..d074f902d1 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -348,7 +348,7 @@ bool LedgerEntrySet::threadOwners(SLE::ref node, Ledger::ref ledger, return false; } -void LedgerEntrySet::calcRawMeta(Serializer& s) +void LedgerEntrySet::calcRawMeta(Serializer& s, TER result) { // calculate the raw meta data and return it. This must be called before the set is committed // Entries modified only as a result of building the transaction metadata @@ -476,7 +476,7 @@ void LedgerEntrySet::calcRawMeta(Serializer& s) cLog(lsINFO) << "Metadata:" << mSet.getJson(0); #endif - mSet.addRaw(s); + mSet.addRaw(s, result); } // <-- uNodeDir: For deletion, present to make dirDelete efficient. diff --git a/src/LedgerEntrySet.h b/src/LedgerEntrySet.h index 11192c363d..2558f12881 100644 --- a/src/LedgerEntrySet.h +++ b/src/LedgerEntrySet.h @@ -120,7 +120,7 @@ public: STAmount accountFunds(const uint160& uAccountID, const STAmount& saDefault); Json::Value getJson(int) const; - void calcRawMeta(Serializer&); + void calcRawMeta(Serializer&, TER result); // iterator functions bool isEmpty() const { return mEntries.empty(); } diff --git a/src/TransactionEngine.cpp b/src/TransactionEngine.cpp index 36712c6a30..167b275bab 100644 --- a/src/TransactionEngine.cpp +++ b/src/TransactionEngine.cpp @@ -457,7 +457,7 @@ TER TransactionEngine::applyTransaction(const SerializedTransaction& txn, Transa { // Transaction succeeded fully or (retries are not allowed and the transaction succeeded partially). Serializer m; - mNodes.calcRawMeta(m); + mNodes.calcRawMeta(m, terResult); txnWrite(); diff --git a/src/TransactionMeta.cpp b/src/TransactionMeta.cpp index 33840cc108..213542dd0f 100644 --- a/src/TransactionMeta.cpp +++ b/src/TransactionMeta.cpp @@ -7,13 +7,18 @@ #include TransactionMetaSet::TransactionMetaSet(const uint256& txid, uint32 ledger, const std::vector& vec) : - mTransactionID(txid), mLedger(ledger), mNodes(sfTransactionMetaData) + mTransactionID(txid), mLedger(ledger), mNodes(sfAffectedNodes) { Serializer s(vec); SerializerIterator sit(s); - std::auto_ptr obj = STArray::deserialize(sit, sfTransactionMetaData); - mNodes = * static_cast(obj.get()); + std::auto_ptr pobj = STObject::deserialize(sit, sfAffectedNodes); + STObject *obj = static_cast(pobj.get()); + if (!obj) + throw std::runtime_error("bad metadata"); + + mResult = obj->getFieldU8(sfTransactionResult); + mNodes = * dynamic_cast(&obj->getField(sfAffectedNodes)); } bool TransactionMetaSet::isNodeAffected(const uint256& node) const @@ -57,7 +62,7 @@ void TransactionMetaSet::init(const uint256& id, uint32 ledger) { mTransactionID = id; mLedger = ledger; - mNodes = STArray(sfTransactionMetaData); + mNodes = STArray(sfAffectedNodes); } void TransactionMetaSet::swap(TransactionMetaSet& s) @@ -85,10 +90,28 @@ static bool compare(const STObject& o1, const STObject& o2) return o1.getFieldH256(sfLedgerIndex) < o2.getFieldH256(sfLedgerIndex); } -void TransactionMetaSet::addRaw(Serializer& s) +STObject TransactionMetaSet::getAsObject() const { + STObject metaData(sfTransactionMetaData); + metaData.setFieldU8(sfTransactionResult, mResult); + metaData.addObject(mNodes); + return metaData; +} + +void TransactionMetaSet::addRaw(Serializer& s, TER result) +{ + mResult = 255; + + if (result == tesSUCCESS) + mResult = 0; + else if (result == tepPATH_DRY) + mResult = 1; + else if (result == tepPATH_PARTIAL) + mResult = 2; + mNodes.sort(compare); - mNodes.add(s); + + getAsObject().add(s); } // vim:ts=4 diff --git a/src/TransactionMeta.h b/src/TransactionMeta.h index 90623b8de7..78f3378f5b 100644 --- a/src/TransactionMeta.h +++ b/src/TransactionMeta.h @@ -12,6 +12,7 @@ #include "Serializer.h" #include "SerializedTypes.h" #include "SerializedObject.h" +#include "TransactionErr.h" class TransactionMetaSet { @@ -19,13 +20,14 @@ public: typedef boost::shared_ptr pointer; protected: - uint256 mTransactionID; - uint32 mLedger; + uint256 mTransactionID; + uint32 mLedger; + int mResult; STArray mNodes; public: - TransactionMetaSet() : mLedger(0) { ; } + TransactionMetaSet() : mLedger(0), mResult(255) { ; } TransactionMetaSet(const uint256& txID, uint32 ledger) : mTransactionID(txID), mLedger(ledger) { ; } TransactionMetaSet(const uint256& txID, uint32 ledger, const std::vector&); @@ -40,8 +42,10 @@ public: STObject& getAffectedNode(const uint256&, SField::ref type, bool overrideType); const STObject& peekAffectedNode(const uint256&) const; - Json::Value getJson(int p) const { return mNodes.getJson(p); } - void addRaw(Serializer&); + Json::Value getJson(int p) const { return getAsObject().getJson(p); } + void addRaw(Serializer&, TER); + + STObject getAsObject() const; static bool thread(STObject& node, const uint256& prevTxID, uint32 prevLgrID); };