From 5ac22ff31b552e744b3792fafed8e31eab087fc4 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 20 Nov 2012 12:09:51 -0800 Subject: [PATCH 1/5] Two more load monitoring hooks. --- src/cpp/ripple/JobQueue.cpp | 1 + src/cpp/ripple/JobQueue.h | 1 + src/cpp/ripple/LedgerConsensus.cpp | 9 ++++++--- src/cpp/ripple/LedgerConsensus.h | 3 ++- src/cpp/ripple/TransactionMaster.cpp | 8 +++++++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/JobQueue.cpp b/src/cpp/ripple/JobQueue.cpp index a72e8ddae..dd730dcb6 100644 --- a/src/cpp/ripple/JobQueue.cpp +++ b/src/cpp/ripple/JobQueue.cpp @@ -23,6 +23,7 @@ const char* Job::toString(JobType t) case jtCLIENT: return "clientCommand"; case jtPEER: return "peerCommand"; case jtDISK: return "diskAccess"; + case jtLEDGER: return "acceptLedger"; default: assert(false); return "unknown"; } } diff --git a/src/cpp/ripple/JobQueue.h b/src/cpp/ripple/JobQueue.h index 339804d20..1101cc2f7 100644 --- a/src/cpp/ripple/JobQueue.h +++ b/src/cpp/ripple/JobQueue.h @@ -36,6 +36,7 @@ enum JobType jtCLIENT = 10, jtPEER = 11, jtDISK = 12, + jtLEDGER = 13, }; #define NUM_JOB_TYPES 16 diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index 8daade3bd..7cc01870d 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -1021,9 +1021,12 @@ void LedgerConsensus::beginAccept(bool synchronous) theApp->getOPs().newLCL(mPeerPositions.size(), mCurrentMSeconds, mNewLedgerHash); if (synchronous) - accept(consensusSet); + accept(consensusSet, LoadEvent::pointer()); else - theApp->getIOService().post(boost::bind(&LedgerConsensus::accept, shared_from_this(), consensusSet)); + { + theApp->getIOService().post(boost::bind(&LedgerConsensus::accept, shared_from_this(), consensusSet, + theApp->getJobQueue().getLoadEvent(jtLEDGER))); + } } void LedgerConsensus::playbackProposals() @@ -1170,7 +1173,7 @@ uint32 LedgerConsensus::roundCloseTime(uint32 closeTime) return closeTime - (closeTime % mCloseResolution); } -void LedgerConsensus::accept(SHAMap::ref set) +void LedgerConsensus::accept(SHAMap::ref set, LoadEvent::pointer) { boost::recursive_mutex::scoped_lock masterLock(theApp->getMasterLock()); assert(set->getHash() == mOurPosition->getCurrentHash()); diff --git a/src/cpp/ripple/LedgerConsensus.h b/src/cpp/ripple/LedgerConsensus.h index 3b66fa9f4..1db706996 100644 --- a/src/cpp/ripple/LedgerConsensus.h +++ b/src/cpp/ripple/LedgerConsensus.h @@ -18,6 +18,7 @@ #include "CanonicalTXSet.h" #include "TransactionEngine.h" #include "InstanceCounter.h" +#include "LoadMonitor.h" DEFINE_INSTANCE(LedgerConsensus); @@ -120,7 +121,7 @@ protected: boost::unordered_set mDeadNodes; // final accept logic - void accept(SHAMap::ref txSet); + void accept(SHAMap::ref txSet, LoadEvent::pointer); void weHave(const uint256& id, Peer::ref avoidPeer); void startAcquiring(const TransactionAcquire::pointer&); diff --git a/src/cpp/ripple/TransactionMaster.cpp b/src/cpp/ripple/TransactionMaster.cpp index 196845bfd..bfa0a8b72 100644 --- a/src/cpp/ripple/TransactionMaster.cpp +++ b/src/cpp/ripple/TransactionMaster.cpp @@ -53,13 +53,19 @@ SerializedTransaction::pointer TransactionMaster::fetch(SHAMapItem::ref item, bo return txn; } +static void saveTransactionHelper(Transaction::pointer txn, LoadEvent::pointer) +{ + Transaction::saveTransaction(txn); +} + bool TransactionMaster::canonicalize(Transaction::pointer& txn, bool may_be_new) { uint256 tid = txn->getID(); if (!tid) return false; if (mCache.canonicalize(tid, txn)) return true; if (may_be_new) - theApp->getAuxService().post(boost::bind(&Transaction::saveTransaction, txn)); + theApp->getAuxService().post(boost::bind(&saveTransactionHelper, txn, + theApp->getJobQueue().getLoadEvent(jtDISK))); return false; } // vim:ts=4 From 61df47658c0bff798c5e6df25c63cd7fdf13e164 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 20 Nov 2012 12:54:47 -0800 Subject: [PATCH 2/5] Fixed huge performance-sapping bug. --- src/cpp/ripple/HashedObject.cpp | 46 +++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 7505a4c24..68ba89e8d 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -84,30 +84,32 @@ void HashedObjectStore::bulkWrite() fAdd("INSERT INTO CommittedObjects (Hash,ObjType,LedgerIndex,Object) VALUES ('%s','%c','%u',%s);"); Database* db = theApp->getHashNodeDB()->getDB(); - ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); - - db->executeSQL("BEGIN TRANSACTION;"); - - BOOST_FOREACH(const boost::shared_ptr& it, set) { - if (!SQL_EXISTS(db, boost::str(fExists % it->getHash().GetHex()))) - { - char type; - switch(it->getType()) - { - case hotLEDGER: type = 'L'; break; - case hotTRANSACTION: type = 'T'; break; - case hotACCOUNT_NODE: type = 'A'; break; - case hotTRANSACTION_NODE: type = 'N'; break; - default: type = 'U'; - } - std::string rawData; - db->escape(&(it->getData().front()), it->getData().size(), rawData); - db->executeSQL(boost::str(fAdd % it->getHash().GetHex() % type % it->getIndex() % rawData )); - } - } + ScopedLock sl = theApp->getHashNodeDB()->getDBLock(); - db->executeSQL("END TRANSACTION;"); + db->executeSQL("BEGIN TRANSACTION;"); + + BOOST_FOREACH(const boost::shared_ptr& it, set) + { + if (!SQL_EXISTS(db, boost::str(fExists % it->getHash().GetHex()))) + { + char type; + switch(it->getType()) + { + case hotLEDGER: type = 'L'; break; + case hotTRANSACTION: type = 'T'; break; + case hotACCOUNT_NODE: type = 'A'; break; + case hotTRANSACTION_NODE: type = 'N'; break; + default: type = 'U'; + } + std::string rawData; + db->escape(&(it->getData().front()), it->getData().size(), rawData); + db->executeSQL(boost::str(fAdd % it->getHash().GetHex() % type % it->getIndex() % rawData )); + } + } + + db->executeSQL("END TRANSACTION;"); + } } } From b4ff61b9e6a614991eae5a3629f7400f30190973 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 20 Nov 2012 12:57:25 -0800 Subject: [PATCH 3/5] Fix the other half. --- src/cpp/ripple/HashedObject.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 68ba89e8d..7765574ad 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -117,7 +117,6 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash) { HashedObject::pointer obj; { - ScopedLock sl(theApp->getHashNodeDB()->getDBLock()); obj = mCache.fetch(hash); if (obj) { From 7d4d18bf8de1e86d0b81a11488bfec51a1bbfac9 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 20 Nov 2012 13:04:17 -0800 Subject: [PATCH 4/5] Add RPC to load tracking. --- src/cpp/ripple/JobQueue.cpp | 1 + src/cpp/ripple/JobQueue.h | 1 + src/cpp/ripple/RPCHandler.cpp | 2 ++ 3 files changed, 4 insertions(+) diff --git a/src/cpp/ripple/JobQueue.cpp b/src/cpp/ripple/JobQueue.cpp index dd730dcb6..e8840537b 100644 --- a/src/cpp/ripple/JobQueue.cpp +++ b/src/cpp/ripple/JobQueue.cpp @@ -24,6 +24,7 @@ const char* Job::toString(JobType t) case jtPEER: return "peerCommand"; case jtDISK: return "diskAccess"; case jtLEDGER: return "acceptLedger"; + case jtRPC: return "rpc"; default: assert(false); return "unknown"; } } diff --git a/src/cpp/ripple/JobQueue.h b/src/cpp/ripple/JobQueue.h index 1101cc2f7..4f0ce559c 100644 --- a/src/cpp/ripple/JobQueue.h +++ b/src/cpp/ripple/JobQueue.h @@ -37,6 +37,7 @@ enum JobType jtPEER = 11, jtDISK = 12, jtLEDGER = 13, + jtRPC = 14, }; #define NUM_JOB_TYPES 16 diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index ba9f6f78b..a9564872c 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -1318,6 +1318,8 @@ Json::Value RPCHandler::doCommand(const std::string& command, Json::Value& param cLog(lsTRACE) << "RPC:" << command; cLog(lsTRACE) << "RPC params:" << params; + LoadEvent::pointer le = theApp->getJobQueue().getLoadEvent(jtRPC); + mRole = role; static struct { From 0f010f04452e15373b7c1b64524fbf1897add5ab Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 20 Nov 2012 14:23:10 -0800 Subject: [PATCH 5/5] Close a loophole where someone requests a large number of proofs of work while they're very easy and then has unlimited use of our system for several minutes because they can keep turning in the easy proofs of work, causing us to keep raising the proof of work level to insane levels. --- src/cpp/ripple/ProofOfWork.cpp | 42 +++++++++++++++++++++++++++------- src/cpp/ripple/ProofOfWork.h | 2 ++ 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/src/cpp/ripple/ProofOfWork.cpp b/src/cpp/ripple/ProofOfWork.cpp index 724362059..c550a9749 100644 --- a/src/cpp/ripple/ProofOfWork.cpp +++ b/src/cpp/ripple/ProofOfWork.cpp @@ -109,12 +109,9 @@ bool ProofOfWork::checkSolution(const uint256& solution) const return getSHA512Half(buf2) <= mTarget; } -ProofOfWorkGenerator::ProofOfWorkGenerator() : - mIterations(128), - mTarget("0003FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"), - mLastDifficultyChange(time(NULL)), - mValidTime(180) +ProofOfWorkGenerator::ProofOfWorkGenerator() : mValidTime(180) { + setDifficulty(1); RAND_bytes(mSecret.begin(), mSecret.size()); } @@ -162,6 +159,8 @@ POWResult ProofOfWorkGenerator::checkProof(const std::string& token, const uint2 time_t t = lexical_cast_s(fields[3]); time_t now = time(NULL); + int iterations = lexical_cast_s(fields[2]); + { boost::mutex::scoped_lock sl(mLock); if ((t * 4) > (now + mValidTime)) @@ -169,10 +168,15 @@ POWResult ProofOfWorkGenerator::checkProof(const std::string& token, const uint2 cLog(lsDEBUG) << "PoW " << token << " has expired"; return powEXPIRED; } + + if (((iterations != mIterations) || (target != mTarget)) && getPowEntry(target, iterations) < (mPowEntry - 2)) + { // difficulty has increased more than two times since PoW requested + cLog(lsINFO) << "Difficulty has increased since PoW requested"; + return powTOOEASY; + } } - - ProofOfWork pow(token, lexical_cast_s(fields[2]), challenge, target); + ProofOfWork pow(token, iterations, challenge, target); if (!pow.checkSolution(solution)) { cLog(lsDEBUG) << "PoW " << token << " has a bad nonce"; @@ -181,7 +185,6 @@ POWResult ProofOfWorkGenerator::checkProof(const std::string& token, const uint2 { boost::mutex::scoped_lock sl(mLock); -// if (...) return powTOOEASY; if (!mSolvedChallenges.insert(powMap_vt(now, challenge)).second) { cLog(lsDEBUG) << "PoW " << token << " has been reused"; @@ -208,6 +211,16 @@ void ProofOfWorkGenerator::sweep() } while(1); } +void ProofOfWorkGenerator::loadHigh() +{ + // WRITEME +} + +void ProofOfWorkGenerator::loadLow() +{ + // WRITEME +} + struct PowEntry { const char *target; @@ -256,6 +269,19 @@ PowEntry PowEntries[31] = { "00003FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF", 262144}, // 77309411328, 8 MB }; +int ProofOfWorkGenerator::getPowEntry(const uint256& target, int iterations) +{ + for (int i = 0; i < 31; ++i) + if (PowEntries[i].iterations == iterations) + { + uint256 t; + t.SetHex(PowEntries[i].target); + if (t == target) + return i; + } + return -1; +} + void ProofOfWorkGenerator::setDifficulty(int i) { assert((i >= 0) && (i <= 30)); diff --git a/src/cpp/ripple/ProofOfWork.h b/src/cpp/ripple/ProofOfWork.h index 3d88ffd19..7c77f8659 100644 --- a/src/cpp/ripple/ProofOfWork.h +++ b/src/cpp/ripple/ProofOfWork.h @@ -78,6 +78,8 @@ public: void loadHigh(); void loadLow(); void sweep(void); + + static int getPowEntry(const uint256& target, int iterations); }; #endif