diff --git a/src/Application.cpp b/src/Application.cpp index 236f46ebf1..ec56e6b914 100644 --- a/src/Application.cpp +++ b/src/Application.cpp @@ -17,6 +17,8 @@ #include #include +SETUP_LOG(); + Application* theApp = NULL; DatabaseCon::DatabaseCon(const std::string& strName, const char *initStrings[], int initCount) @@ -56,7 +58,7 @@ void Application::stop() mValidations.flush(); mAuxService.stop(); - Log(lsINFO) << "Stopped: " << mIOService.stopped(); + cLog(lsINFO) << "Stopped: " << mIOService.stopped(); } static void InitDB(DatabaseCon** dbCon, const char *fileName, const char *dbInit[], int dbCount) @@ -90,12 +92,12 @@ void Application::run() if (theConfig.START_UP == Config::FRESH) { - Log(lsINFO) << "Starting new Ledger"; + cLog(lsINFO) << "Starting new Ledger"; startNewLedger(); } else if (theConfig.START_UP == Config::LOAD) { - Log(lsINFO) << "Loading Old Ledger"; + cLog(lsINFO) << "Loading Old Ledger"; loadOldLedger(); } else if (theConfig.START_UP == Config::NETWORK) @@ -155,7 +157,7 @@ void Application::run() if (theConfig.RUN_STANDALONE) { - Log(lsWARNING) << "Running in standalone mode"; + cLog(lsWARNING) << "Running in standalone mode"; mNetOps.setStandAlone(); } else @@ -185,8 +187,8 @@ void Application::startNewLedger() NewcoinAddress rootAddress = NewcoinAddress::createAccountPublic(rootGeneratorMaster, 0); // Print enough information to be able to claim root account. - Log(lsINFO) << "Root master seed: " << rootSeedMaster.humanSeed(); - Log(lsINFO) << "Root account: " << rootAddress.humanAccountID(); + cLog(lsINFO) << "Root master seed: " << rootSeedMaster.humanSeed(); + cLog(lsINFO) << "Root account: " << rootAddress.humanAccountID(); { Ledger::pointer firstLedger = boost::make_shared(rootAddress, SYSTEM_CURRENCY_START); @@ -218,13 +220,34 @@ void Application::loadOldLedger() } lastLedger->setClosed(); + cLog(lsINFO) << "Loading ledger " << lastLedger->getHash() << " seq:" << lastLedger->getLedgerSeq(); + + if (lastLedger->getAccountHash().isZero()) + { + cLog(lsFATAL) << "Ledger is empty."; + assert(false); + exit(-1); + } + + if (!lastLedger->walkLedger()) + { + cLog(lsFATAL) << "Ledger is missing nodes."; + exit(-1); + } + + if (!lastLedger->assertSane()) + { + cLog(lsFATAL) << "Ledger is not sane."; + exit(-1); + } + Ledger::pointer openLedger = boost::make_shared(false, boost::ref(*lastLedger)); mMasterLedger.switchLedgers(lastLedger, openLedger); mNetOps.setLastCloseTime(lastLedger->getCloseTimeNC()); } catch (SHAMapMissingNode& mn) { - Log(lsFATAL) << "Cannot load ledger. " << mn; + cLog(lsFATAL) << "Cannot load ledger. " << mn; exit(-1); } } diff --git a/src/HashedObject.cpp b/src/HashedObject.cpp index 25bff66268..837a518a66 100644 --- a/src/HashedObject.cpp +++ b/src/HashedObject.cpp @@ -21,7 +21,11 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false=already in cache, true = added to cache assert(hash == Serializer::getSHA512Half(data)); - if (!theApp->getHashNodeDB()) return true; + if (!theApp->getHashNodeDB()) + { + cLog(lsTRACE) << "HOS: no db"; + return true; + } if (mCache.touch(hash)) { cLog(lsTRACE) << "HOS: " << hash << " store: incache"; @@ -31,59 +35,70 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index, HashedObject::pointer object = boost::make_shared(type, index, data, hash); if (!mCache.canonicalize(hash, object)) { +// cLog(lsTRACE) << "Queuing write for " << hash; boost::recursive_mutex::scoped_lock sl(mWriteMutex); mWriteSet.push_back(object); - if (!mWritePending && (mWriteSet.size() >= 64)) + if (!mWritePending) { mWritePending = true; boost::thread t(boost::bind(&HashedObjectStore::bulkWrite, this)); t.detach(); } } +// else +// cLog(lsTRACE) << "HOS: already had " << hash; return true; } void HashedObjectStore::bulkWrite() { std::vector< boost::shared_ptr > set; - set.reserve(128); - + while (1) { - boost::recursive_mutex::scoped_lock sl(mWriteMutex); - mWriteSet.swap(set); - mWritePending = false; - } - cLog(lsINFO) << "HOS: BulkWrite " << set.size(); + set.clear(); + set.reserve(128); - static boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); - static boost::format - fAdd("INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); - - Database* db = theApp->getHashNodeDB()->getDB(); - ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); - - db->executeSQL("BEGIN TRANSACTION;"); - - BOOST_FOREACH(const boost::shared_ptr& it, set) - { - if (!SQL_EXISTS(db, boost::str(fExists % it->getHash().GetHex()))) { - char type; - switch(it->getType()) + boost::recursive_mutex::scoped_lock sl(mWriteMutex); + mWriteSet.swap(set); + if (set.empty()) { - case hotLEDGER: type= 'L'; break; - case hotTRANSACTION: type = 'T'; break; - case hotACCOUNT_NODE: type = 'A'; break; - case hotTRANSACTION_NODE: type = 'N'; break; - default: type = 'U'; + mWritePending = false; + return; } - std::string rawData; - db->escape(&(it->getData().front()), it->getData().size(), rawData); - db->executeSQL(boost::str(fAdd % it->getHash().GetHex() % type % it->getIndex() % rawData )); } - } +// cLog(lsINFO) << "HOS: writing " << set.size(); - db->executeSQL("END TRANSACTION;"); + static boost::format fExists("SELECT ObjType FROM CommittedObjects WHERE Hash = '%s';"); + static boost::format + fAdd("INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); + + Database* db = theApp->getHashNodeDB()->getDB(); + ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); + + db->executeSQL("BEGIN TRANSACTION;"); + + BOOST_FOREACH(const boost::shared_ptr& it, set) + { + if (!SQL_EXISTS(db, boost::str(fExists % it->getHash().GetHex()))) + { + char type; + switch(it->getType()) + { + case hotLEDGER: type = 'L'; break; + case hotTRANSACTION: type = 'T'; break; + case hotACCOUNT_NODE: type = 'A'; break; + case hotTRANSACTION_NODE: type = 'N'; break; + default: type = 'U'; + } + std::string rawData; + db->escape(&(it->getData().front()), it->getData().size(), rawData); + db->executeSQL(boost::str(fAdd % it->getHash().GetHex() % type % it->getIndex() % rawData )); + } + } + + db->executeSQL("END TRANSACTION;"); + } } HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) @@ -111,7 +126,7 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) if (!db->executeSQL(sql) || !db->startIterRows()) { - cLog(lsTRACE) << "HOS: " << hash << " fetch: not in db"; +// cLog(lsTRACE) << "HOS: " << hash << " fetch: not in db"; return HashedObject::pointer(); } diff --git a/src/Ledger.cpp b/src/Ledger.cpp index cda250cf28..b21958e58c 100644 --- a/src/Ledger.cpp +++ b/src/Ledger.cpp @@ -31,11 +31,12 @@ Ledger::Ledger(const NewcoinAddress& masterID, uint64 startAmount) : mTotCoins(s AccountState::pointer startAccount = boost::make_shared(masterID); startAccount->peekSLE().setFieldAmount(sfBalance, startAmount); startAccount->peekSLE().setFieldU32(sfSequence, 1); + cLog(lsTRACE) << "root account: " << startAccount->peekSLE().getJson(0); + + mAccountStateMap->armDirty(); writeBack(lepCREATE, startAccount->getSLE()); -#if 0 - std::cerr << "Root account:"; - startAccount->dump(); -#endif + mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq); + mAccountStateMap->disarmDirty(); } Ledger::Ledger(const uint256 &parentHash, const uint256 &transHash, const uint256 &accountHash, @@ -225,18 +226,24 @@ bool Ledger::addTransaction(const uint256& txID, const Serializer& txn) { // low-level - just add to table SHAMapItem::pointer item = boost::make_shared(txID, txn.peekData()); if (!mTransactionMap->addGiveItem(item, true, false)) + { + cLog(lsWARNING) << "Attempt to add transaction to ledger that already had it"; return false; + } return true; } bool Ledger::addTransaction(const uint256& txID, const Serializer& txn, const Serializer& md) { // low-level - just add to table - Serializer s(txn.getDataLength() + md.getDataLength() + 64); + Serializer s(txn.getDataLength() + md.getDataLength() + 16); s.addVL(txn.peekData()); s.addVL(md.peekData()); SHAMapItem::pointer item = boost::make_shared(txID, s.peekData()); if (!mTransactionMap->addGiveItem(item, true, true)) + { + cLog(lsFATAL) << "Attempt to add transaction+MD to ledger that already had it"; return false; + } return true; } @@ -273,6 +280,22 @@ Transaction::pointer Ledger::getTransaction(const uint256& transID) const return txn; } +SerializedTransaction::pointer Ledger::getSTransaction(SHAMapItem::ref item, SHAMapTreeNode::TNType type) +{ + SerializerIterator sit(item->peekSerializer()); + + if (type == SHAMapTreeNode::tnTRANSACTION_NM) + return boost::make_shared(boost::ref(sit)); + else if (type == SHAMapTreeNode::tnTRANSACTION_MD) + { + Serializer sTxn(sit.getVL()); + SerializerIterator tSit(sTxn); + return boost::make_shared(boost::ref(tSit)); + } + + return SerializedTransaction::pointer(); +} + bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, TransactionMetaSet::pointer& meta) { SHAMapTreeNode::TNType type; @@ -285,7 +308,7 @@ bool Ledger::getTransaction(const uint256& txID, Transaction::pointer& txn, Tran txn = theApp->getMasterTransaction().fetch(txID, false); meta = TransactionMetaSet::pointer(); if (!txn) - txn = Transaction::sharedTransaction(item->getData(), true); + txn = Transaction::sharedTransaction(item->peekData(), true); } else if (type == SHAMapTreeNode::tnTRANSACTION_MD) { // in tree with metadata @@ -313,12 +336,13 @@ bool Ledger::unitTest() uint256 Ledger::getHash() { - if(!mValidHash) updateHash(); - return(mHash); + if (!mValidHash) + updateHash(); + return mHash; } -void Ledger::saveAcceptedLedger(Ledger::ref ledger) -{ +void Ledger::saveAcceptedLedger() +{ // can be called in a different thread static boost::format ledgerExists("SELECT LedgerSeq FROM Ledgers where LedgerSeq = %d;"); static boost::format deleteLedger("DELETE FROM Ledgers WHERE LedgerSeq = %d;"); static boost::format AcctTransExists("SELECT LedgerSeq FROM AccountTransactions WHERE TransId = '%s';"); @@ -328,78 +352,92 @@ void Ledger::saveAcceptedLedger(Ledger::ref ledger) "(LedgerHash,LedgerSeq,PrevHash,TotalCoins,ClosingTime,PrevClosingTime,CloseTimeRes,CloseFlags," "AccountSetHash,TransSetHash) VALUES ('%s','%u','%s','%s','%u','%u','%d','%u','%s','%s');"); - ScopedLock sl(theApp->getLedgerDB()->getDBLock()); - if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % ledger->mLedgerSeq))) - theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % ledger->mLedgerSeq)); - theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % - ledger->getHash().GetHex() % ledger->mLedgerSeq % ledger->mParentHash.GetHex() % - boost::lexical_cast(ledger->mTotCoins) % ledger->mCloseTime % ledger->mParentCloseTime % - ledger->mCloseResolution % ledger->mCloseFlags % - ledger->mAccountHash.GetHex() % ledger->mTransHash.GetHex())); - - // write out dirty nodes - int fc; - while ((fc = ledger->mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, ledger->mLedgerSeq)) > 0) - { cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; } - while ((fc = ledger->mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, ledger->mLedgerSeq)) > 0) - { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } - ledger->disarmDirty(); - - SHAMap& txSet = *ledger->peekTransactionMap(); - Database *db = theApp->getTxnDB()->getDB(); - ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); - db->executeSQL("BEGIN TRANSACTION;"); - for (SHAMapItem::pointer item = txSet.peekFirstItem(); !!item; item = txSet.peekNextItem(item->getTag())) + if (!getAccountHash().isNonZero()) { - SerializedTransaction::pointer txn = theApp->getMasterTransaction().fetch(item, false, ledger->mLedgerSeq); - - // Make sure transaction is in AccountTransactions. - if (!SQL_EXISTS(db, boost::str(AcctTransExists % item->getTag().GetHex()))) - { - // Transaction not in AccountTransactions - std::vector accts = txn->getAffectedAccounts(); - - std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq) VALUES "; - bool first = true; - for (std::vector::iterator it = accts.begin(), end = accts.end(); it != end; ++it) - { - if (!first) - sql += ", ('"; - else - { - sql += "('"; - first = false; - } - sql += txn->getTransactionID().GetHex(); - sql += "','"; - sql += it->humanAccountID(); - sql += "',"; - sql += boost::lexical_cast(ledger->getLedgerSeq()); - sql += ")"; - } - sql += ";"; - Log(lsTRACE) << "ActTx: " << sql; - db->executeSQL(sql); // may already be in there - } - - if (SQL_EXISTS(db, boost::str(transExists % txn->getTransactionID().GetHex()))) - { - // In Transactions, update LedgerSeq and Status. - db->executeSQL(boost::str(updateTx - % ledger->getLedgerSeq() - % TXN_SQL_VALIDATED - % txn->getTransactionID().GetHex())); - } - else - { - // Not in Transactions, insert the whole thing.. - db->executeSQL( - txn->getSQLInsertHeader() + txn->getSQL(ledger->getLedgerSeq(), TXN_SQL_VALIDATED) + ";"); - } + cLog(lsFATAL) << "AH is zero: " << getJson(0); + assert(false); } - db->executeSQL("COMMIT TRANSACTION;"); - theApp->getOPs().pubLedger(ledger); + assert (getAccountHash() == mAccountStateMap->getHash()); + assert (getTransHash() == mTransactionMap->getHash()); + + { + ScopedLock sl(theApp->getLedgerDB()->getDBLock()); + if (SQL_EXISTS(theApp->getLedgerDB()->getDB(), boost::str(ledgerExists % mLedgerSeq))) + theApp->getLedgerDB()->getDB()->executeSQL(boost::str(deleteLedger % mLedgerSeq)); + theApp->getLedgerDB()->getDB()->executeSQL(boost::str(addLedger % + getHash().GetHex() % mLedgerSeq % mParentHash.GetHex() % + boost::lexical_cast(mTotCoins) % mCloseTime % mParentCloseTime % + mCloseResolution % mCloseFlags % + mAccountHash.GetHex() % mTransHash.GetHex())); + + // write out dirty nodes + int fc; + while ((fc = mTransactionMap->flushDirty(256, hotTRANSACTION_NODE, mLedgerSeq)) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty transaction nodes"; } + while ((fc = mAccountStateMap->flushDirty(256, hotACCOUNT_NODE, mLedgerSeq)) > 0) + { cLog(lsINFO) << "Flushed " << fc << " dirty state nodes"; } + disarmDirty(); + + SHAMap& txSet = *peekTransactionMap(); + Database *db = theApp->getTxnDB()->getDB(); + ScopedLock dbLock = theApp->getTxnDB()->getDBLock(); + db->executeSQL("BEGIN TRANSACTION;"); + SHAMapTreeNode::TNType type; + for (SHAMapItem::pointer item = txSet.peekFirstItem(type); !!item; + item = txSet.peekNextItem(item->getTag(), type)) + { + SerializedTransaction::pointer txn = getSTransaction(item, type); + assert(txn); + + // Make sure transaction is in AccountTransactions. + if (!SQL_EXISTS(db, boost::str(AcctTransExists % item->getTag().GetHex()))) + { + // Transaction not in AccountTransactions + std::vector accts = txn->getAffectedAccounts(); + + std::string sql = "INSERT INTO AccountTransactions (TransID, Account, LedgerSeq) VALUES "; + bool first = true; + for (std::vector::iterator it = accts.begin(), end = accts.end(); it != end; ++it) + { + if (!first) + sql += ", ('"; + else + { + sql += "('"; + first = false; + } + sql += txn->getTransactionID().GetHex(); + sql += "','"; + sql += it->humanAccountID(); + sql += "',"; + sql += boost::lexical_cast(getLedgerSeq()); + sql += ")"; + } + sql += ";"; + Log(lsTRACE) << "ActTx: " << sql; + db->executeSQL(sql); // may already be in there + } + + if (SQL_EXISTS(db, boost::str(transExists % txn->getTransactionID().GetHex()))) + { + // In Transactions, update LedgerSeq and Status. + db->executeSQL(boost::str(updateTx + % getLedgerSeq() + % TXN_SQL_VALIDATED + % txn->getTransactionID().GetHex())); + } + else + { + // Not in Transactions, insert the whole thing.. + db->executeSQL( + txn->getSQLInsertHeader() + txn->getSQL(getLedgerSeq(), TXN_SQL_VALIDATED) + ";"); + } + } + db->executeSQL("COMMIT TRANSACTION;"); + } + + theApp->getOPs().pubLedger(shared_from_this()); } Ledger::pointer Ledger::getSQL(const std::string& sql) @@ -449,6 +487,7 @@ Ledger::pointer Ledger::getSQL(const std::string& sql) assert(false); return Ledger::pointer(); } + Log(lsDEBUG) << "Loaded ledger: " << ledgerHash; return ret; } @@ -469,6 +508,11 @@ Ledger::pointer Ledger::loadByHash(const uint256& ledgerHash) } void Ledger::addJson(Json::Value& ret, int options) +{ + ret["ledger"] = getJson(options); +} + +Json::Value Ledger::getJson(int options) { Json::Value ledger(Json::objectValue); @@ -554,8 +598,8 @@ void Ledger::addJson(Json::Value& ret, int options) } ledger["accountState"] = state; } - ledger["seqNum"]=boost::lexical_cast(mLedgerSeq); - ret["ledger"] = ledger; + ledger["seqNum"] = boost::lexical_cast(mLedgerSeq); + return ledger; } void Ledger::setAcquiring(void) @@ -613,7 +657,7 @@ LedgerStateParms Ledger::writeBack(LedgerStateParms parms, SLE::ref entry) if (create) { assert(!mAccountStateMap->hasItem(entry->getIndex())); - if(!mAccountStateMap->addGiveItem(item, false, false)) // FIXME: TX metadata + if(!mAccountStateMap->addGiveItem(item, false, false)) { assert(false); return lepERROR; @@ -621,7 +665,7 @@ LedgerStateParms Ledger::writeBack(LedgerStateParms parms, SLE::ref entry) return lepCREATED; } - if (!mAccountStateMap->updateGiveItem(item, false, false)) // FIXME: TX metadata + if (!mAccountStateMap->updateGiveItem(item, false, false)) { assert(false); return lepERROR; @@ -930,6 +974,32 @@ uint256 Ledger::getRippleStateIndex(const NewcoinAddress& naA, const NewcoinAddr return s.getSHA512Half(); } +bool Ledger::walkLedger() +{ + std::vector missingNodes; + mAccountStateMap->walkMap(missingNodes, 32); + if (sLog(lsINFO) && !missingNodes.empty()) + { + Log(lsINFO) << missingNodes.size() << " missing account node(s)"; + Log(lsINFO) << "First: " << missingNodes[0]; + } + mTransactionMap->walkMap(missingNodes, 32); + return missingNodes.empty(); +} +bool Ledger::assertSane() +{ + if (mHash.isNonZero() && mAccountHash.isNonZero() && mAccountStateMap && mTransactionMap && + (mAccountHash == mAccountStateMap->getHash()) && (mTransHash == mTransactionMap->getHash())) + return true; + + Log(lsFATAL) << "ledger is not sane"; + Json::Value j = getJson(0); + j["accountTreeHash"] = mAccountHash.GetHex(); + j["transTreeHash"] = mTransHash.GetHex(); + + assert(false); + return false; +} // vim:ts=4 diff --git a/src/Ledger.h b/src/Ledger.h index bf362fb6cc..0c201d12fe 100644 --- a/src/Ledger.h +++ b/src/Ledger.h @@ -149,6 +149,7 @@ public: bool hasTransaction(const uint256& TransID) const { return mTransactionMap->hasItem(TransID); } Transaction::pointer getTransaction(const uint256& transID) const; bool getTransaction(const uint256& transID, Transaction::pointer& txn, TransactionMetaSet::pointer& txMeta); + static SerializedTransaction::pointer getSTransaction(SHAMapItem::ref, SHAMapTreeNode::TNType); // high-level functions AccountState::pointer getAccountState(const NewcoinAddress& acctID); @@ -157,7 +158,7 @@ public: SLE::pointer getAccountRoot(const NewcoinAddress& naAccountID); // database functions - static void saveAcceptedLedger(Ledger::ref); + void saveAcceptedLedger(); static Ledger::pointer loadByIndex(uint32 ledgerIndex); static Ledger::pointer loadByHash(const uint256& ledgerHash); @@ -277,8 +278,12 @@ public: SLE::pointer getRippleState(const uint160& uiA, const uint160& uiB, const uint160& uCurrency) { return getRippleState(getRippleStateIndex(NewcoinAddress::createAccountID(uiA), NewcoinAddress::createAccountID(uiB), uCurrency)); } + Json::Value getJson(int options); void addJson(Json::Value&, int options); + bool walkLedger(); + bool assertSane(); + static bool unitTest(); }; diff --git a/src/LedgerConsensus.cpp b/src/LedgerConsensus.cpp index e8e9e06850..83dcbefefe 100644 --- a/src/LedgerConsensus.cpp +++ b/src/LedgerConsensus.cpp @@ -58,7 +58,6 @@ void TransactionAcquire::trigger(Peer::ref peer, bool timer) } if (!mHaveRoot) { - cLog(lsINFO) << "have no root"; ripple::TMGetLedger tmGL; tmGL.set_ledgerhash(mHash.begin(), mHash.size()); tmGL.set_itype(ripple::liTS_CANDIDATE); @@ -301,37 +300,9 @@ void LedgerConsensus::checkLCL() void LedgerConsensus::handleLCL(const uint256& lclHash) { - mPrevLedgerHash = lclHash; - if (mPreviousLedger->getHash() == mPrevLedgerHash) - return; - - Ledger::pointer newLCL = theApp->getMasterLedger().getLedgerByHash(lclHash); - if (newLCL) - mPreviousLedger = newLCL; - else if (mAcquiringLedger && (mAcquiringLedger->getHash() == mPrevLedgerHash)) - return; - else - { - cLog(lsWARNING) << "Need consensus ledger " << mPrevLedgerHash; - - mAcquiringLedger = theApp->getMasterLedgerAcquire().findCreate(mPrevLedgerHash); - std::vector peerList = theApp->getConnectionPool().getPeerVector(); - - bool found = false; - BOOST_FOREACH(Peer::ref peer, peerList) - { - if (peer->hasLedger(mPrevLedgerHash)) - { - found = true; - mAcquiringLedger->peerHas(peer); - } - } - - if (!found) - { - BOOST_FOREACH(Peer::ref peer, peerList) - mAcquiringLedger->peerHas(peer); - } + if (mPrevLedgerHash != lclHash) + { // first time switching to this ledger + mPrevLedgerHash = lclHash; if (mHaveCorrectLCL && mProposing && mOurPosition) { @@ -339,15 +310,47 @@ void LedgerConsensus::handleLCL(const uint256& lclHash) mOurPosition->bowOut(); propose(); } - mHaveCorrectLCL = false; mProposing = false; mValidating = false; - mCloseTimes.clear(); mPeerPositions.clear(); + mPeerData.clear(); mDisputes.clear(); + mCloseTimes.clear(); mDeadNodes.clear(); playbackProposals(); - return; + } + + if (mPreviousLedger->getHash() != mPrevLedgerHash) + { // we need to switch the ledger we're working from + Ledger::pointer newLCL = theApp->getMasterLedger().getLedgerByHash(lclHash); + if (newLCL) + mPreviousLedger = newLCL; + else if (!mAcquiringLedger || (mAcquiringLedger->getHash() != mPrevLedgerHash)) + { // need to start acquiring the correct consensus LCL + cLog(lsWARNING) << "Need consensus ledger " << mPrevLedgerHash; + + mAcquiringLedger = theApp->getMasterLedgerAcquire().findCreate(mPrevLedgerHash); + std::vector peerList = theApp->getConnectionPool().getPeerVector(); + + bool found = false; + BOOST_FOREACH(Peer::ref peer, peerList) + { + if (peer->hasLedger(mPrevLedgerHash)) + { + found = true; + mAcquiringLedger->peerHas(peer); + } + } + + if (!found) + { + BOOST_FOREACH(Peer::ref peer, peerList) + mAcquiringLedger->peerHas(peer); + } + + mHaveCorrectLCL = false; + return; + } } cLog(lsINFO) << "Acquired the consensus ledger " << mPrevLedgerHash; @@ -356,7 +359,6 @@ void LedgerConsensus::handleLCL(const uint256& lclHash) mCloseResolution = ContinuousLedgerTiming::getNextLedgerTimeResolution( mPreviousLedger->getCloseResolution(), mPreviousLedger->getCloseAgree(), mPreviousLedger->getLedgerSeq() + 1); - playbackProposals(); } void LedgerConsensus::takeInitialPosition(Ledger& initialLedger) @@ -675,7 +677,7 @@ void LedgerConsensus::updateOurPositions() for (std::map::iterator it = closeTimes.begin(), end = closeTimes.end(); it != end; ++it) { cLog(lsINFO) << "CCTime: " << it->first << " has " << it->second << ", " << thresh << " required"; - if (it->second > thresh) + if (it->second >= thresh) { cLog(lsINFO) << "Close time consensus reached: " << it->first; mHaveCloseTimeConsensus = true; @@ -1114,6 +1116,15 @@ void LedgerConsensus::accept(SHAMap::ref set) newLCL->setAccepted(closeTime, mCloseResolution, closeTimeCorrect); newLCL->updateHash(); uint256 newLCLHash = newLCL->getHash(); + + if (sLog(lsTRACE)) + { + Log(lsTRACE) << "newLCL"; + Json::Value p; + newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); + Log(lsTRACE) << p; + } + statusChange(ripple::neACCEPTED_LEDGER, *newLCL); if (mValidating) { @@ -1181,13 +1192,6 @@ void LedgerConsensus::accept(SHAMap::ref set) theApp->getOPs().closeTimeOffset(offset); } - if (sLog(lsTRACE)) - { - Log(lsTRACE) << "newLCL"; - Json::Value p; - newLCL->addJson(p, LEDGER_JSON_DUMP_TXNS | LEDGER_JSON_DUMP_STATE); - Log(lsTRACE) << p; - } } void LedgerConsensus::endConsensus() diff --git a/src/LedgerEntrySet.cpp b/src/LedgerEntrySet.cpp index b1b4ed750c..e804f33c25 100644 --- a/src/LedgerEntrySet.cpp +++ b/src/LedgerEntrySet.cpp @@ -310,7 +310,8 @@ bool LedgerEntrySet::threadTx(const NewcoinAddress& threadTo, Ledger::ref ledger return threadTx(sle, ledger, newMods); } -bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, boost::unordered_map& newMods) +bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, + boost::unordered_map& newMods) { // node = the node that was modified/deleted/created // threadTo = the node that needs to know uint256 prevTxID; @@ -323,7 +324,8 @@ bool LedgerEntrySet::threadTx(SLE::ref threadTo, Ledger::ref ledger, boost::unor return false; } -bool LedgerEntrySet::threadOwners(SLE::ref node, Ledger::ref ledger, boost::unordered_map& newMods) +bool LedgerEntrySet::threadOwners(SLE::ref node, Ledger::ref ledger, + boost::unordered_map& newMods) { // thread new or modified node to owner or owners if (node->hasOneOwner()) // thread to owner's account { diff --git a/src/LedgerHistory.cpp b/src/LedgerHistory.cpp index 6dd3a9c4a3..2c1563d639 100644 --- a/src/LedgerHistory.cpp +++ b/src/LedgerHistory.cpp @@ -36,6 +36,7 @@ void LedgerHistory::addAcceptedLedger(Ledger::pointer ledger) assert(ledger->isAccepted()); assert(ledger->isImmutable()); mLedgersByIndex.insert(std::make_pair(ledger->getLedgerSeq(), ledger)); + boost::thread thread(boost::bind(&Ledger::saveAcceptedLedger, ledger)); thread.detach(); } diff --git a/src/Peer.cpp b/src/Peer.cpp index 8fdeea74ba..99558dd847 100644 --- a/src/Peer.cpp +++ b/src/Peer.cpp @@ -1035,7 +1035,10 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet) if ((!ledger) || (packet.has_ledgerseq() && (packet.ledgerseq() != ledger->getLedgerSeq()))) { punishPeer(PP_UNKNOWN_REQUEST); - cLog(lsWARNING) << "Can't find the ledger they want"; + if (ledger) + cLog(lsWARNING) << "Ledger has wrong sequence"; + else + cLog(lsWARNING) << "Can't find the ledger they want"; return; } @@ -1056,7 +1059,7 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet) { // new-style root request cLog(lsINFO) << "Ledger root w/map roots request"; SHAMap::pointer map = ledger->peekAccountStateMap(); - if (map) + if (map && map->getHash().isNonZero()) { // return account state root node if possible Serializer rootNode(768); if (map->getRootNode(rootNode, snfWIRE)) @@ -1065,7 +1068,7 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet) if (ledger->getTransHash().isNonZero()) { map = ledger->peekTransactionMap(); - if (map) + if (map && map->getHash().isNonZero()) { rootNode.resize(0); if (map->getRootNode(rootNode, snfWIRE)) diff --git a/src/SHAMap.cpp b/src/SHAMap.cpp index 09877fa403..890075215e 100644 --- a/src/SHAMap.cpp +++ b/src/SHAMap.cpp @@ -230,11 +230,13 @@ void SHAMap::returnNode(SHAMapTreeNode::pointer& node, bool modify) assert(node->getSeq() <= mSeq); if (node && modify && (node->getSeq() != mSeq)) { // have a CoW - if (mDirtyNodes) (*mDirtyNodes)[*node] = node; node = boost::make_shared(*node, mSeq); + if (mDirtyNodes) + (*mDirtyNodes)[*node] = node; assert(node->isValid()); mTNByID[*node] = node; - if (node->isRoot()) root = node; + if (node->isRoot()) + root = node; } } @@ -549,7 +551,7 @@ bool SHAMap::delItem(const uint256& id) return true; } -bool SHAMap::addGiveItem(const SHAMapItem::pointer& item, bool isTransaction, bool hasMeta) +bool SHAMap::addGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta) { // add the specified item, does not update #ifdef ST_DEBUG std::cerr << "aGI " << item->getTag() << std::endl; @@ -647,7 +649,7 @@ bool SHAMap::addItem(const SHAMapItem& i, bool isTransaction, bool hasMetaData) return addGiveItem(boost::make_shared(i), isTransaction, hasMetaData); } -bool SHAMap::updateGiveItem(const SHAMapItem::pointer& item, bool isTransaction, bool hasMeta) +bool SHAMap::updateGiveItem(SHAMapItem::ref item, bool isTransaction, bool hasMeta) { // can't change the tag but can change the hash uint256 tag = item->getTag(); @@ -686,12 +688,15 @@ void SHAMapItem::dump() SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const uint256& hash) { if (!theApp->running()) + { + cLog(lsTRACE) << "Trying to fetch external node with application not running"; throw SHAMapMissingNode(mType, id, hash); + } HashedObject::pointer obj(theApp->getHashedObjectStore().retrieve(hash)); if (!obj) { - Log(lsTRACE) << "fetchNodeExternal: missing " << hash; +// Log(lsTRACE) << "fetchNodeExternal: missing " << hash; throw SHAMapMissingNode(mType, id, hash); } assert(Serializer::getSHA512Half(obj->getData()) == hash); @@ -699,9 +704,7 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui try { SHAMapTreeNode::pointer ret = boost::make_shared(id, obj->getData(), mSeq, snfPREFIX); -#ifdef DEBUG assert((ret->getNodeHash() == hash) && (id == *ret)); -#endif return ret; } catch (...) @@ -713,9 +716,18 @@ SHAMapTreeNode::pointer SHAMap::fetchNodeExternal(const SHAMapNode& id, const ui void SHAMap::fetchRoot(const uint256& hash) { + if (sLog(lsTRACE)) + { + if (mType == smtTRANSACTION) + Log(lsTRACE) << "Fetch root TXN node " << hash; + else if (mType == smtSTATE) + Log(lsTRACE) << "Fetch root STATE node " << hash; + else + Log(lsTRACE) << "Fetch root SHAMap node " << hash; + } root = fetchNodeExternal(SHAMapNode(), hash); - root->makeInner(); mTNByID[*root] = root; + assert(root->getNodeHash() == hash); } void SHAMap::armDirty() @@ -735,8 +747,8 @@ int SHAMap::flushDirty(int maxNodes, HashedObjectType t, uint32 seq) boost::unordered_map::iterator it = dirtyNodes.begin(); while (it != dirtyNodes.end()) { - tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first; - tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first; +// tLog(mType == smtTRANSACTION, lsDEBUG) << "TX node write " << it->first; +// tLog(mType == smtSTATE, lsDEBUG) << "STATE node write " << it->first; s.erase(); it->second->addRaw(s, snfPREFIX); theApp->getHashedObjectStore().store(t, seq, s.peekData(), s.getSHA512Half()); diff --git a/src/SHAMap.h b/src/SHAMap.h index f144b5ca9f..6d579a918d 100644 --- a/src/SHAMap.h +++ b/src/SHAMap.h @@ -195,6 +195,7 @@ public: bool isInnerNode() const { return !mItem; } bool setChildHash(int m, const uint256& hash); bool isEmptyBranch(int m) const { return !mHashes[m]; } + bool isEmpty() const; int getBranchCount() const; void makeInner(); const uint256& getChildHash(int m) const @@ -400,6 +401,8 @@ public: static std::vector checkTrustedPath(const uint256& ledgerHash, const uint256& leafIndex, const std::list >& path); + void walkMap(std::vector& missingNodes, int maxMissing); + bool deepCompare(SHAMap& other); virtual void dump(bool withHashes = false); }; diff --git a/src/SHAMapDiff.cpp b/src/SHAMapDiff.cpp index d0e78e8f98..05619df210 100644 --- a/src/SHAMapDiff.cpp +++ b/src/SHAMapDiff.cpp @@ -186,3 +186,38 @@ bool SHAMap::compare(SHAMap::ref otherMap, SHAMapDiff& differences, int maxCount return true; } + +void SHAMap::walkMap(std::vector& missingNodes, int maxMissing) +{ + std::stack nodeStack; + + boost::recursive_mutex::scoped_lock sl(mLock); + + if (!root->isInner()) // root is only node, and we have it + return; + + nodeStack.push(root); + + while (!nodeStack.empty()) + { + SHAMapTreeNode::pointer node = nodeStack.top(); + nodeStack.pop(); + + for (int i = 0; i < 16; ++i) + if (!node->isEmptyBranch(i)) + { + try + { + SHAMapTreeNode::pointer d = getNode(node->getChildNodeID(i), node->getChildHash(i), false); + if (d->isInner()) + nodeStack.push(d); + } + catch (SHAMapMissingNode& n) + { + missingNodes.push_back(n); + if (--maxMissing <= 0) + return; + } + } + } +} diff --git a/src/SHAMapNodes.cpp b/src/SHAMapNodes.cpp index 859b47382e..6c02b79a3b 100644 --- a/src/SHAMapNodes.cpp +++ b/src/SHAMapNodes.cpp @@ -278,6 +278,8 @@ SHAMapTreeNode::SHAMapTreeNode(const SHAMapNode& id, const std::vectorpeekData()); } - else assert(false); + else + assert(false); - if (nh == mHash) return false; + if (nh == mHash) + return false; mHash = nh; return true; } @@ -357,10 +361,12 @@ bool SHAMapTreeNode::updateHash() void SHAMapTreeNode::addRaw(Serializer& s, SHANodeFormat format) { assert((format == snfPREFIX) || (format == snfWIRE)); - if (mType == tnERROR) throw std::runtime_error("invalid I node type"); + if (mType == tnERROR) + throw std::runtime_error("invalid I node type"); if (mType == tnINNER) { + assert(!isEmpty()); if (format == snfPREFIX) { s.add32(sHP_InnerNode); @@ -449,6 +455,14 @@ SHAMapItem::pointer SHAMapTreeNode::getItem() const return boost::make_shared(*mItem); } +bool SHAMapTreeNode::isEmpty() const +{ + assert(isInner()); + for (int i = 0; i < 16; ++i) + if (mHashes[i].isNonZero()) return false; + return true; +} + int SHAMapTreeNode::getBranchCount() const { assert(isInner()); @@ -513,9 +527,9 @@ bool SHAMapTreeNode::setChildHash(int m, const uint256 &hash) std::ostream& operator<<(std::ostream& out, const SHAMapMissingNode& mn) { if (mn.getMapType() == smtTRANSACTION) - out << "Missing/TXN(" << mn.getNodeID() << ")"; + out << "Missing/TXN(" << mn.getNodeID() << "/" << mn.getNodeHash() << ")"; else if (mn.getMapType() == smtSTATE) - out << "Missing/STA(" << mn.getNodeID() << ")"; + out << "Missing/STA(" << mn.getNodeID() << "/" << mn.getNodeHash() << ")"; else out << "Missing/" << mn.getNodeID(); return out; diff --git a/src/SerializedLedger.cpp b/src/SerializedLedger.cpp index acc58da3c1..1e29a99c22 100644 --- a/src/SerializedLedger.cpp +++ b/src/SerializedLedger.cpp @@ -98,10 +98,12 @@ bool SerializedLedgerEntry::thread(const uint256& txID, uint32 ledgerSeq, uint25 uint256 oldPrevTxID = getFieldH256(sfLastTxnID); Log(lsTRACE) << "Thread Tx:" << txID << " prev:" << oldPrevTxID; if (oldPrevTxID == txID) + { // this transaction is already threaded + assert(getFieldU32(sfLastTxnSeq) == ledgerSeq); return false; + } prevTxID = oldPrevTxID; prevLedgerID = getFieldU32(sfLastTxnSeq); - assert(prevTxID != txID); setFieldH256(sfLastTxnID, txID); setFieldU32(sfLastTxnSeq, ledgerSeq); return true; diff --git a/src/SerializedObject.cpp b/src/SerializedObject.cpp index 4aa7d067b2..678b1add3c 100644 --- a/src/SerializedObject.cpp +++ b/src/SerializedObject.cpp @@ -13,6 +13,8 @@ #include "TransactionFormats.h" #include "SerializedTransaction.h" +SETUP_LOG(); + std::auto_ptr STObject::makeDefaultObject(SerializedTypeID id, SField::ref name) { assert((id == STI_NOTPRESENT) || (id == name.fieldType)); @@ -154,7 +156,7 @@ bool STObject::setType(const std::vector &type) { if (elem->flags != SOE_OPTIONAL) { - Log(lsWARNING) << "setType !valid missing " << elem->e_field.fieldName; + cLog(lsWARNING) << "setType !valid missing " << elem->e_field.fieldName; valid = false; } newData.push_back(makeNonPresentObject(elem->e_field)); @@ -168,7 +170,7 @@ bool STObject::setType(const std::vector &type) { if (!t.getFName().isDiscardable()) { - Log(lsWARNING) << "setType !valid leftover: " << t.getFName().getName(); + cLog(lsWARNING) << "setType !valid leftover: " << t.getFName().getName(); valid = false; } } @@ -214,7 +216,7 @@ bool STObject::set(SerializerIterator& sit, int depth) SField::ref fn = SField::getField(type, field); if (fn.isInvalid()) { - Log(lsWARNING) << "Unknown field: field_type=" << type << ", field_name=" << field; + cLog(lsWARNING) << "Unknown field: field_type=" << type << ", field_name=" << field; throw std::runtime_error("Unknown field"); } giveObject(makeDeserializedObject(fn.fieldType, fn, sit, depth + 1)); @@ -853,7 +855,7 @@ STArray* STArray::construct(SerializerIterator& sit, SField::ref field) SField::ref fn = SField::getField(type, field); if (fn.isInvalid()) { - Log(lsTRACE) << "Unknown field: " << type << "/" << field; + cLog(lsTRACE) << "Unknown field: " << type << "/" << field; throw std::runtime_error("Unknown field"); } @@ -1097,7 +1099,7 @@ std::auto_ptr STObject::parseJson(const Json::Value& object, SField::r NewcoinAddress a; if (!a.setAccountID(strValue)) { - Log(lsINFO) << "Invalid acccount JSON: " << fieldName << ": " << strValue; + cLog(lsINFO) << "Invalid acccount JSON: " << fieldName << ": " << strValue; throw std::runtime_error("Account invalid"); } data.push_back(new STAccount(field, a.getAccountID())); @@ -1163,8 +1165,8 @@ BOOST_AUTO_TEST_CASE( FieldManipulation_test ) if (object1.getSerializer() == object2.getSerializer()) { - Log(lsINFO) << "O1: " << object1.getJson(0); - Log(lsINFO) << "O2: " << object2.getJson(0); + cLog(lsINFO) << "O1: " << object1.getJson(0); + cLog(lsINFO) << "O2: " << object2.getJson(0); BOOST_FAIL("STObject error 4"); } object1.makeFieldAbsent(sfTestH256); diff --git a/src/Serializer.cpp b/src/Serializer.cpp index 78067feeb4..efce6138c2 100644 --- a/src/Serializer.cpp +++ b/src/Serializer.cpp @@ -9,6 +9,9 @@ #include #include "key.h" +#include "Log.h" + +SETUP_LOG(); int Serializer::addZeros(size_t uBytes) { @@ -185,7 +188,10 @@ int Serializer::addFieldID(int type, int name) bool Serializer::getFieldID(int& type, int& name, int offset) const { if (!get8(type, offset)) + { + cLog(lsWARNING) << "gFID: unable to get type"; return false; + } name = type & 15; type >>= 4; if (type == 0) @@ -193,14 +199,20 @@ bool Serializer::getFieldID(int& type, int& name, int offset) const if (!get8(type, ++offset)) return false; if ((type == 0) || (type < 16)) + { + cLog(lsWARNING) << "gFID: uncommon type out of range " << type; return false; + } } if (name == 0) { // uncommon name if (!get8(name, ++offset)) return false; if ((name == 0) || (name < 16)) + { + cLog(lsWARNING) << "gFID: uncommon name out of range " << name; return false; + } } return true; } diff --git a/src/TaggedCache.h b/src/TaggedCache.h index 62a9e699d4..cf1b55bab2 100644 --- a/src/TaggedCache.h +++ b/src/TaggedCache.h @@ -169,7 +169,7 @@ bool TaggedCache::canonicalize(const key_type& key, boost::shared { // in map, but expired. Update in map, insert in cache mit->second = data; mCache.insert(std::make_pair(key, std::make_pair(time(NULL), data))); - return false; + return true; } // in map and cache, canonicalize diff --git a/src/TransactionMeta.cpp b/src/TransactionMeta.cpp index 3f687897c1..3e0cd4035d 100644 --- a/src/TransactionMeta.cpp +++ b/src/TransactionMeta.cpp @@ -195,7 +195,11 @@ bool TransactionMetaNode::thread(const uint256& prevTx, uint32 prevLgr) { BOOST_FOREACH(TransactionMetaNodeEntry& it, mEntries) if (it.getType() == TMSThread) + { + TMNEThread* a = dynamic_cast(&it); + assert(a && (a->getPrevTxID() == prevTx) && (a->getPrevLgr() == prevLgr)); return false; + } addNode(new TMNEThread(prevTx, prevLgr)); return true; } @@ -267,8 +271,7 @@ TransactionMetaSet::TransactionMetaSet(uint32 ledger, const std::vector::iterator it = mNodes.begin(), end = mNodes.end(); - it != end; ++it) + for (std::map::iterator it = mNodes.begin(), end = mNodes.end(); it != end; ++it) it->second.addRaw(s); s.add8(TMNEndOfMetadata); } diff --git a/src/TransactionMeta.h b/src/TransactionMeta.h index e3ac3b6cd4..b329fdb61c 100644 --- a/src/TransactionMeta.h +++ b/src/TransactionMeta.h @@ -81,6 +81,9 @@ public: virtual void addRaw(Serializer&) const; virtual Json::Value getJson(int) const; + const uint256& getPrevTxID() const { return mPrevTxID; } + uint32 getPrevLgr() const { return mPrevLgrSeq; } + protected: virtual TransactionMetaNodeEntry* duplicate(void) const { return new TMNEThread(*this); } virtual int compare(const TransactionMetaNodeEntry&) const; @@ -175,7 +178,7 @@ public: protected: uint256 mTransactionID; uint32 mLedger; - std::map mNodes; + std::map mNodes; // must be an ordered set public: TransactionMetaSet() : mLedger(0) { ; }