From 42e775ebed04446693e5224389a82b203de2dd84 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 30 May 2013 10:09:49 -0700 Subject: [PATCH 01/27] Fix missing dependencies. --- SConstruct | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/SConstruct b/SConstruct index f867ecd20..282693375 100644 --- a/SConstruct +++ b/SConstruct @@ -105,9 +105,11 @@ BOOSTFLAGS = ['-DBOOST_TEST_DYN_LINK', '-DBOOST_FILESYSTEM_NO_DEPRECATED'] env.Append(LINKFLAGS = ['-rdynamic', '-pthread']) env.Append(CCFLAGS = ['-pthread', '-Wall', '-Wno-sign-compare', '-Wno-char-subscripts', '-DSQLITE_THREADSAFE=1']) env.Append(CXXFLAGS = ['-O0', '-pthread', '-Wno-invalid-offsetof', '-Wformat']+BOOSTFLAGS+DEBUGFLAGS) -env.Append(CXXFLAGS = [ '-Isrc/cpp/leveldb', '-Isrc/cpp/leveldb/port', '-Isrc/cpp/leveldb/include', '-DUSE_LEVELDB']) -env.Append(CXXFLAGS = [ '-Ibuild/proto']) -env.Append(CXXFLAGS = [ '-I.', '-Isrc/cpp/ripple']) +env.Append(CXXFLAGS = ['-DUSE_LEVELDB']) + +env.Append(CPPPATH = [ 'src/cpp/leveldb', 'src/cpp/leveldb/port', 'src/cpp/leveldb/include']) +env.Append(CPPPATH = [ 'build/proto']) +env.Append(CPPPATH = [ '.', 'src/cpp/ripple']) if (int(GCC_VERSION[0]) > 4 or (int(GCC_VERSION[0]) == 4 and int(GCC_VERSION[1]) >= 7)): env.Append(CXXFLAGS = ['-std=c++11']) @@ -140,7 +142,7 @@ for dir in [ 'ripple_db', 'ripple_json', 'ripple_ledger', - 'ripple_main', + 'ripple_main', 'ripple_mess', 'ripple_net' ]: From b09df810f76e5897d41a0f69e6e25553f56cac14 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 30 May 2013 15:26:42 -0700 Subject: [PATCH 02/27] "Features" was supposed to be a Vector256. --- src/cpp/ripple/SerializeProto.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/SerializeProto.h b/src/cpp/ripple/SerializeProto.h index a36debe66..45593cce7 100644 --- a/src/cpp/ripple/SerializeProto.h +++ b/src/cpp/ripple/SerializeProto.h @@ -143,6 +143,7 @@ // vector of 256-bit FIELD(Indexes, VECTOR256, 1) FIELD(Hashes, VECTOR256, 2) + FIELD(Features, VECTOR256, 3) // inner object // OBJECT/1 is reserved for end of object @@ -164,6 +165,5 @@ FIELD(Necessary, ARRAY, 6) FIELD(Sufficient, ARRAY, 7) FIELD(AffectedNodes, ARRAY, 8) - FIELD(Features, ARRAY, 9) // vim:ts=4 From b7fea85581c95932be2146d6a007103bfb339f78 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 30 May 2013 12:54:40 -0700 Subject: [PATCH 03/27] Enlarge the immutable SLE cache. --- src/cpp/ripple/Config.cpp | 9 +++++++++ src/cpp/ripple/Config.h | 2 ++ 2 files changed, 11 insertions(+) diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index a4a0b1a92..9aec9de9e 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -512,14 +512,23 @@ void Config::load() int Config::getSize(SizedItemName item) { SizedItem sizeTable[] = { // tiny small medium large huge + { siSweepInterval, { 10, 30, 60, 90, 120 } }, + { siLedgerFetch, { 2, 2, 3, 3, 3 } }, + { siValidationsSize, { 256, 256, 512, 1024, 1024 } }, { siValidationsAge, { 500, 500, 500, 500, 500 } }, + { siNodeCacheSize, { 8192, 65536, 262144, 2097152, 0 } }, { siNodeCacheAge, { 30, 60, 90, 300, 900 } }, + + { siSLECacheSize, { 4096, 8192, 16384, 65536, 0 } }, + { siSLECacheAge, { 30, 60, 90, 120, 300 } }, + { siLedgerSize, { 32, 128, 256, 2048, 0 } }, { siLedgerAge, { 30, 90, 180, 300, 900 } }, + { siHashNodeDBCache, { 4, 12, 24, 32, 64 } }, { siTxnDBCache, { 4, 12, 24, 32, 32 } }, { siLgrDBCache, { 4, 8, 16, 16, 16 } }, diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index 422e9ebb4..007a48fdf 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -57,6 +57,8 @@ enum SizedItemName siValidationsAge, siNodeCacheSize, siNodeCacheAge, + siSLECacheSize, + siSLECacheAge, siLedgerSize, siLedgerAge, siLedgerFetch, From b1646c5b6bf00cd21eb6cd45f3867d38c1127d43 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 30 May 2013 13:07:38 -0700 Subject: [PATCH 04/27] Tune SLE cache size. --- src/cpp/ripple/Application.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 1da62bd3d..ec4439299 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -243,6 +243,9 @@ void Application::setup() mValidations.tune(theConfig.getSize(siValidationsSize), theConfig.getSize(siValidationsAge)); mHashedObjectStore.tune(theConfig.getSize(siNodeCacheSize), theConfig.getSize(siNodeCacheAge)); mLedgerMaster.tune(theConfig.getSize(siLedgerSize), theConfig.getSize(siLedgerAge)); + mSLECache.setTargetSize(theConfig.getSize(siSLECacheSize)); + mSLECache.setTargetAge(theConfig.getSize(siSLECacheAge)); + mLedgerMaster.setMinValidations(theConfig.VALIDATION_QUORUM); #ifdef USE_LEVELDB From b2e1eedbc5551b9fcb9994f01352b10e521b751a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 1 Jun 2013 22:17:26 -0700 Subject: [PATCH 05/27] Fix a cause of deadlocks. Cannot acquire the master lock while holding a PeerSet lock. --- src/cpp/ripple/TransactionAcquire.cpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/cpp/ripple/TransactionAcquire.cpp b/src/cpp/ripple/TransactionAcquire.cpp index 79a309c27..07342084d 100644 --- a/src/cpp/ripple/TransactionAcquire.cpp +++ b/src/cpp/ripple/TransactionAcquire.cpp @@ -13,21 +13,27 @@ TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_A mMap = boost::make_shared(smtTRANSACTION, hash); } -void TransactionAcquire::done() +static void TACompletionHandler(uint256 hash, SHAMap::pointer map) { boost::recursive_mutex::scoped_lock sl(theApp->getMasterLock()); + theApp->getOPs().mapComplete(hash, map); + theApp->getMasterLedgerAcquire().dropLedger(hash); +} + +void TransactionAcquire::done() +{ // We hold a PeerSet lock and so cannot acquire the master lock here + SHAMap::pointer map; if (mFailed) { WriteLog (lsWARNING, TransactionAcquire) << "Failed to acquire TX set " << mHash; - theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); } else { WriteLog (lsINFO, TransactionAcquire) << "Acquired TX set " << mHash; mMap->setImmutable(); - theApp->getOPs().mapComplete(mHash, mMap); + map = mMap; } - theApp->getMasterLedgerAcquire().dropLedger(mHash); + theApp->getIOService().post(boost::bind(&TACompletionHandler, mHash, map)); } void TransactionAcquire::onTimer(bool progress) From 8274432370f8e476e82e1322805ab9afd34d74fe Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 2 Jun 2013 01:48:07 -0700 Subject: [PATCH 06/27] Actually set the cached transaction state. --- src/cpp/ripple/Transactor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/Transactor.cpp b/src/cpp/ripple/Transactor.cpp index be4366d7b..6a7bf4a68 100644 --- a/src/cpp/ripple/Transactor.cpp +++ b/src/cpp/ripple/Transactor.cpp @@ -171,11 +171,11 @@ TER Transactor::preCheck() { if (mTxn.isKnownBad() || (!isSetBit(mParams, tapNO_CHECK_SIGN) && !mTxn.checkSign(mSigningPubKey))) { - mTxn.setGood(); + mTxn.setBad(); WriteLog (lsWARNING, Transactor) << "applyTransaction: Invalid transaction: bad signature"; return temINVALID; } - mTxn.isKnownGood(); + mTxn.setGood(); } return tesSUCCESS; From d1fc9fe94c9323ccd7173f47175040299d396446 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 3 Jun 2013 02:06:02 -0700 Subject: [PATCH 07/27] Filter out secrets from logging. --- modules/ripple_basics/utility/ripple_Log.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/modules/ripple_basics/utility/ripple_Log.cpp b/modules/ripple_basics/utility/ripple_Log.cpp index 0f08d0828..ae7e0ade1 100644 --- a/modules/ripple_basics/utility/ripple_Log.cpp +++ b/modules/ripple_basics/utility/ripple_Log.cpp @@ -54,7 +54,19 @@ Log::~Log() case lsINVALID: assert(false); return; } - logMsg += oss.str(); + std::string ls = oss.str(); + size_t s = ls.find("\"secret\""); + if (s != std::string::npos) + { + s += 8; + size_t sEnd = ls.size() - 1; + if (sEnd > (s + 35)) + sEnd = s + 35; + for (int i = s; i < sEnd; ++i) + ls[i] = '*'; + } + logMsg += ls; + if (logMsg.size() > LOG_MAX_MESSAGE) { logMsg.resize(LOG_MAX_MESSAGE); From d30210ba7b8a4699360994bed4eef5749f43f1ee Mon Sep 17 00:00:00 2001 From: Alex Chung Date: Wed, 5 Jun 2013 01:01:12 -0400 Subject: [PATCH 08/27] Defined OS_MACOSX for OSX build. Was causing LevelDB build fail. --- SConstruct | 1 + 1 file changed, 1 insertion(+) diff --git a/SConstruct b/SConstruct index fe051edb3..b2fd16849 100644 --- a/SConstruct +++ b/SConstruct @@ -117,6 +117,7 @@ if (int(GCC_VERSION[0]) > 4 or (int(GCC_VERSION[0]) == 4 and int(GCC_VERSION[1]) if OSX: env.Append(LINKFLAGS = ['-L/usr/local/opt/openssl/lib']) env.Append(CXXFLAGS = ['-I/usr/local/opt/openssl/include']) + env.Append(CXXFLAGS = ['-DOS_MACOSX']) RIPPLE_SRCS = [ 'src/cpp/database/sqlite3.c', From 963f9279cb5493ccac6c719cf5226efdda3eef11 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 09:27:32 -0700 Subject: [PATCH 09/27] Typo --- src/cpp/ripple/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index fe0664624..35c694abe 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -84,7 +84,7 @@ void Application::stop() mAuxService.stop(); mJobQueue.shutdown(); -#ifdef HAVE_LEVELDB +#ifdef USE_LEVELDB delete mHashNodeLDB: mHashNodeLDB = NULL; #endif From cfc010f08efb8c8193728de257954b13276477f0 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 09:32:33 -0700 Subject: [PATCH 10/27] Typo --- src/cpp/ripple/Application.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 35c694abe..d97493b6a 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -85,7 +85,7 @@ void Application::stop() mJobQueue.shutdown(); #ifdef USE_LEVELDB - delete mHashNodeLDB: + delete mHashNodeLDB; mHashNodeLDB = NULL; #endif From f524484dcb72ebd1d1fd5bec2a560960ea0c95a2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 09:37:26 -0700 Subject: [PATCH 11/27] LevelDB is now mandatory --- src/cpp/ripple/Application.cpp | 15 +-------------- src/cpp/ripple/Application.h | 6 ------ src/cpp/ripple/HashedObject.cpp | 18 ------------------ src/cpp/ripple/HashedObject.h | 6 ------ src/cpp/ripple/UpdateTables.cpp | 2 -- src/cpp/ripple/main.cpp | 2 -- 6 files changed, 1 insertion(+), 48 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index d97493b6a..62cd4bdf5 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -1,10 +1,8 @@ #include "Application.h" -#ifdef USE_LEVELDB #include "leveldb/cache.h" #include "leveldb/filter_policy.h" -#endif #include "AcceptedLedger.h" #include "Config.h" @@ -56,10 +54,7 @@ Application::Application() : mFeatureTable(2 * 7 * 24 * 60 * 60, 200), // two weeks, 200/256 mRpcDB(NULL), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), - mNetNodeDB(NULL), mPathFindDB(NULL), mHashNodeDB(NULL), -#ifdef USE_LEVELDB - mHashNodeLDB(NULL), -#endif + mNetNodeDB(NULL), mPathFindDB(NULL), mHashNodeDB(NULL), mHashNodeLDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL), mWSPublicDoor(NULL), mWSPrivateDoor(NULL), mSweepTimer(mAuxService), mShutdown(false) { @@ -84,10 +79,8 @@ void Application::stop() mAuxService.stop(); mJobQueue.shutdown(); -#ifdef USE_LEVELDB delete mHashNodeLDB; mHashNodeLDB = NULL; -#endif WriteLog (lsINFO, Application) << "Stopped: " << mIOService.stopped(); Instance::shutdown(); @@ -164,7 +157,6 @@ void Application::setup() boost::thread t7(boost::bind(&InitDB, &mPathFindDB, "pathfind.db", PathFindDBInit, PathFindDBCount)); t4.join(); t6.join(); t7.join(); -#ifdef USE_LEVELDB if (mHashedObjectStore.isLevelDB()) { WriteLog (lsINFO, Application) << "LevelDB used for nodes"; @@ -186,7 +178,6 @@ void Application::setup() } } else -#endif { WriteLog (lsINFO, Application) << "SQLite used for nodes"; boost::thread t5(boost::bind(&InitDB, &mHashNodeDB, "hashnode.db", HashNodeDBInit, HashNodeDBCount)); @@ -246,9 +237,7 @@ void Application::setup() mLedgerMaster.setMinValidations(theConfig.VALIDATION_QUORUM); -#ifdef USE_LEVELDB if (!mHashedObjectStore.isLevelDB()) -#endif theApp->getHashNodeDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % (theConfig.getSize(siHashNodeDBCache) * 1024))); @@ -413,9 +402,7 @@ Application::~Application() delete mHashNodeDB; delete mNetNodeDB; delete mPathFindDB; -#ifdef USE_LEVELDB delete mHashNodeLDB; -#endif } void Application::startNewLedger() diff --git a/src/cpp/ripple/Application.h b/src/cpp/ripple/Application.h index e05126e59..0034ec50c 100644 --- a/src/cpp/ripple/Application.h +++ b/src/cpp/ripple/Application.h @@ -1,9 +1,7 @@ #ifndef __APPLICATION__ #define __APPLICATION__ -#ifdef USE_LEVELDB #include "leveldb/db.h" -#endif #include @@ -79,9 +77,7 @@ class Application DatabaseCon *mRpcDB, *mTxnDB, *mLedgerDB, *mWalletDB, *mNetNodeDB, *mPathFindDB, *mHashNodeDB; -#ifdef USE_LEVELDB leveldb::DB *mHashNodeLDB; -#endif ConnectionPool mConnectionPool; PeerDoor* mPeerDoor; @@ -152,9 +148,7 @@ public: DatabaseCon* getPathFindDB() { return mPathFindDB; } DatabaseCon* getHashNodeDB() { return mHashNodeDB; } -#ifdef USE_LEVELDB leveldb::DB* getHashNodeLDB() { return mHashNodeLDB; } -#endif uint256 getNonce256() { return mNonce256; } std::size_t getNonceST() { return mNonceST; } diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 9366be5d9..a158e5590 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -1,9 +1,7 @@ #include "HashedObject.h" -#ifdef USE_LEVELDB #include "leveldb/db.h" #include "leveldb/write_batch.h" -#endif #include #include @@ -31,14 +29,6 @@ HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : WriteLog (lsFATAL, HashedObject) << "Incorrect database selection"; assert(false); } -#ifndef USE_LEVELDB - if (mLevelDB) - { - WriteLog (lsFATAL) << "LevelDB has been selected but not compiled"; - assert(false); - } -#endif - } void HashedObjectStore::tune(int size, int age) @@ -61,8 +51,6 @@ int HashedObjectStore::getWriteLoad() return std::max(mWriteLoad, static_cast(mWriteSet.size())); } -#ifdef USE_LEVELDB - bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false = already in cache, true = added to cache @@ -177,8 +165,6 @@ HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) return obj; } -#endif - bool HashedObjectStore::storeSQLite(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false = already in cache, true = added to cache @@ -405,8 +391,6 @@ HashedObject::pointer HashedObjectStore::retrieveSQLite(const uint256& hash) return obj; } -#ifdef USE_LEVELDB - int HashedObjectStore::import(const std::string& file) { WriteLog (lsWARNING, HashedObject) << "Hashed object import from \"" << file << "\"."; @@ -476,6 +460,4 @@ int HashedObjectStore::import(const std::string& file) return count; } -#endif - // vim:ts=4 diff --git a/src/cpp/ripple/HashedObject.h b/src/cpp/ripple/HashedObject.h index 3240e0421..afc678284 100644 --- a/src/cpp/ripple/HashedObject.h +++ b/src/cpp/ripple/HashedObject.h @@ -82,19 +82,15 @@ public: bool store(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { -#ifdef USE_LEVELDB if (mLevelDB) return storeLevelDB(type, index, data, hash); -#endif return storeSQLite(type, index, data, hash); } HashedObject::pointer retrieve(const uint256& hash) { -#ifdef USE_LEVELDB if (mLevelDB) return retrieveLevelDB(hash); -#endif return retrieveSQLite(hash); } @@ -103,12 +99,10 @@ public: HashedObject::pointer retrieveSQLite(const uint256& hash); void bulkWriteSQLite(Job&); -#ifdef USE_LEVELDB bool storeLevelDB(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); HashedObject::pointer retrieveLevelDB(const uint256& hash); void bulkWriteLevelDB(Job&); -#endif void waitWrite(); diff --git a/src/cpp/ripple/UpdateTables.cpp b/src/cpp/ripple/UpdateTables.cpp index 9a31b360c..12143c9a5 100644 --- a/src/cpp/ripple/UpdateTables.cpp +++ b/src/cpp/ripple/UpdateTables.cpp @@ -113,7 +113,6 @@ void Application::updateTables(bool ldbImport) exit(1); } -#ifdef USE_LEVELDB if (theApp->getHashedObjectStore().isLevelDB()) { boost::filesystem::path hashPath = theConfig.DATA_DIR / "hashnode.db"; @@ -134,5 +133,4 @@ void Application::updateTables(bool ldbImport) } } } -#endif } diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index 24dd35441..0ef94a4d8 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -150,9 +150,7 @@ int main(int argc, char* argv[]) ("start", "Start from a fresh Ledger.") ("net", "Get the initial ledger from the network.") ("fg", "Run in the foreground.") -#ifdef USE_LEVELDB ("import", "Import SQLite node DB into LevelDB.") -#endif ; // Interpret positional arguments as --parameters. From f141b79d43dae9672d0a4fbac933e96193c0bf69 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 10:25:03 -0700 Subject: [PATCH 12/27] Implement the ephemeral cache. --- src/cpp/ripple/Application.cpp | 18 ++++- src/cpp/ripple/Application.h | 2 + src/cpp/ripple/Config.cpp | 2 + src/cpp/ripple/Config.h | 1 + src/cpp/ripple/HashedObject.cpp | 137 +++++++++++++++++++++++--------- src/cpp/ripple/HashedObject.h | 2 +- 6 files changed, 123 insertions(+), 39 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 62cd4bdf5..a4bc761b9 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -54,7 +54,7 @@ Application::Application() : mFeatureTable(2 * 7 * 24 * 60 * 60, 200), // two weeks, 200/256 mRpcDB(NULL), mTxnDB(NULL), mLedgerDB(NULL), mWalletDB(NULL), - mNetNodeDB(NULL), mPathFindDB(NULL), mHashNodeDB(NULL), mHashNodeLDB(NULL), + mNetNodeDB(NULL), mPathFindDB(NULL), mHashNodeDB(NULL), mHashNodeLDB(NULL), mEphemeralLDB(NULL), mConnectionPool(mIOService), mPeerDoor(NULL), mRPCDoor(NULL), mWSPublicDoor(NULL), mWSPrivateDoor(NULL), mSweepTimer(mAuxService), mShutdown(false) { @@ -82,6 +82,9 @@ void Application::stop() delete mHashNodeLDB; mHashNodeLDB = NULL; + delete mEphemeralLDB; + mEphemeralLDB = NULL; + WriteLog (lsINFO, Application) << "Stopped: " << mIOService.stopped(); Instance::shutdown(); } @@ -176,6 +179,18 @@ void Application::setup() StopSustain(); exit(3); } + + if (!theConfig.LDB_EPHEMERAL.empty()) + { + leveldb::Status status = leveldb::DB::Open(options, theConfig.LDB_EPHEMERAL, &mEphemeralLDB); + if (!status.ok() || !mEphemeralLDB) + { + WriteLog(lsFATAL, Application) << "Unable to open/create epehemeral db: " + << theConfig.LDB_EPHEMERAL << " " << status.ToString(); + StopSustain(); + exit(3); + } + } } else { @@ -403,6 +418,7 @@ Application::~Application() delete mNetNodeDB; delete mPathFindDB; delete mHashNodeLDB; + delete mEphemeralLDB; } void Application::startNewLedger() diff --git a/src/cpp/ripple/Application.h b/src/cpp/ripple/Application.h index 0034ec50c..14d222921 100644 --- a/src/cpp/ripple/Application.h +++ b/src/cpp/ripple/Application.h @@ -78,6 +78,7 @@ class Application DatabaseCon *mRpcDB, *mTxnDB, *mLedgerDB, *mWalletDB, *mNetNodeDB, *mPathFindDB, *mHashNodeDB; leveldb::DB *mHashNodeLDB; + leveldb::DB *mEphemeralLDB; ConnectionPool mConnectionPool; PeerDoor* mPeerDoor; @@ -149,6 +150,7 @@ public: DatabaseCon* getHashNodeDB() { return mHashNodeDB; } leveldb::DB* getHashNodeLDB() { return mHashNodeLDB; } + leveldb::DB* getEphemeralLDB() { return mEphemeralLDB; } uint256 getNonce256() { return mNonce256; } std::size_t getNonceST() { return mNonceST; } diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index b695650af..c73de571d 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -24,6 +24,7 @@ #define SECTION_FEE_ACCOUNT_RESERVE "fee_account_reserve" #define SECTION_FEE_OWNER_RESERVE "fee_owner_reserve" #define SECTION_NODE_DB "node_db" +#define SECTION_LDB_EPHEMERAL "ephemeral_db" #define SECTION_LEDGER_HISTORY "ledger_history" #define SECTION_IPS "ips" #define SECTION_NETWORK_QUORUM "network_quorum" @@ -357,6 +358,7 @@ void Config::load() (void) sectionSingleB(secConfig, SECTION_RPC_PASSWORD, RPC_PASSWORD); (void) sectionSingleB(secConfig, SECTION_RPC_USER, RPC_USER); (void) sectionSingleB(secConfig, SECTION_NODE_DB, NODE_DB); + (void) sectionSingleB(secConfig, SECTION_LDB_EPHEMERAL, LDB_EPHEMERAL); if (sectionSingleB(secConfig, SECTION_RPC_PORT, strTemp)) RPC_PORT = boost::lexical_cast(strTemp); diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index 581c7126e..fa9dca3da 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -85,6 +85,7 @@ public: boost::filesystem::path DEBUG_LOGFILE; boost::filesystem::path VALIDATORS_FILE; // As specifed in rippled.cfg. std::string NODE_DB; // Database to use for nodes + std::string LDB_EPHEMERAL; // Database for temporary storage bool LDB_IMPORT; // Import into LevelDB bool ELB_SUPPORT; // Support Amazon ELB diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index a158e5590..096b9546c 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -16,7 +16,7 @@ DECLARE_INSTANCE(HashedObject); HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : mCache("HashedObjectStore", cacheSize, cacheAge), mNegativeCache("HashedObjectNegativeCache", 0, 120), - mWriteGeneration(0), mWriteLoad(0), mWritePending(false), mLevelDB(false) + mWriteGeneration(0), mWriteLoad(0), mWritePending(false), mLevelDB(false), mEphemeralDB(false) { mWriteSet.reserve(128); @@ -29,6 +29,8 @@ HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : WriteLog (lsFATAL, HashedObject) << "Incorrect database selection"; assert(false); } + if (!theConfig.LDB_EPHEMERAL.empty()) + mEphemeralDB = true; } void HashedObjectStore::tune(int size, int age) @@ -51,6 +53,74 @@ int HashedObjectStore::getWriteLoad() return std::max(mWriteLoad, static_cast(mWriteSet.size())); } +static HashedObject::pointer LLRetrieve(const uint256& hash, leveldb::DB* db) +{ // low-level retrieve + std::string sData; + + leveldb::Status st = db->Get(leveldb::ReadOptions(), + leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), &sData); + if (!st.ok()) + { + assert(st.IsNotFound()); + return HashedObject::pointer(); + } + + const unsigned char* bufPtr = reinterpret_cast(&sData[0]); + uint32 index = htonl(*reinterpret_cast(bufPtr)); + int htype = bufPtr[8]; + + return boost::make_shared(static_cast(htype), index, + bufPtr + 9, sData.size() - 9, hash); +} + +static void LLWrite(boost::shared_ptr ptr, leveldb::DB* db) +{ // low-level write single + HashedObject& obj = *ptr; + std::vector rawData(9 + obj.mData.size()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast(bufPtr + 0) = ntohl(obj.mLedgerIndex); + *reinterpret_cast(bufPtr + 4) = ntohl(obj.mLedgerIndex); + *(bufPtr + 8) = static_cast(obj.mType); + memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size()); + + leveldb::Status st = db->Put(leveldb::WriteOptions(), + leveldb::Slice(reinterpret_cast(obj.mHash.begin()), obj.mHash.size()), + leveldb::Slice(reinterpret_cast(bufPtr), rawData.size())); + if (!st.ok()) + { + WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; + assert(false); + } +} + +static void LLWrite(const std::vector< boost::shared_ptr >& set, leveldb::DB* db) +{ // low-level write set + leveldb::WriteBatch batch; + + BOOST_FOREACH(const boost::shared_ptr& it, set) + { + const HashedObject& obj = *it; + std::vector rawData(9 + obj.mData.size()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast(bufPtr + 0) = ntohl(obj.mLedgerIndex); + *reinterpret_cast(bufPtr + 4) = ntohl(obj.mLedgerIndex); + *(bufPtr + 8) = static_cast(obj.mType); + memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size()); + + batch.Put(leveldb::Slice(reinterpret_cast(obj.mHash.begin()), obj.mHash.size()), + leveldb::Slice(reinterpret_cast(bufPtr), rawData.size())); + } + + leveldb::Status st = db->Write(leveldb::WriteOptions(), &batch); + if (!st.ok()) + { + WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; + assert(false); + } +} + bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false = already in cache, true = added to cache @@ -106,31 +176,9 @@ void HashedObjectStore::bulkWriteLevelDB(Job &) setSize = set.size(); } - { - leveldb::WriteBatch batch; - - BOOST_FOREACH(const boost::shared_ptr& it, set) - { - const HashedObject& obj = *it; - std::vector rawData(9 + obj.mData.size()); - unsigned char* bufPtr = &rawData.front(); - - *reinterpret_cast(bufPtr + 0) = ntohl(obj.mLedgerIndex); - *reinterpret_cast(bufPtr + 4) = ntohl(obj.mLedgerIndex); - *(bufPtr + 8) = static_cast(obj.mType); - memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size()); - - batch.Put(leveldb::Slice(reinterpret_cast(obj.mHash.begin()), obj.mHash.size()), - leveldb::Slice(reinterpret_cast(bufPtr), rawData.size())); - } - - leveldb::Status st = theApp->getHashNodeLDB()->Write(leveldb::WriteOptions(), &batch); - if (!st.ok()) - { - WriteLog (lsFATAL, HashedObject) << "Failed to store hash node"; - assert(false); - } - } + LLWrite(set, theApp->getHashNodeLDB()); + if (mEphemeralDB) + LLWrite(set, theApp->getEphemeralLDB()); } } @@ -140,27 +188,28 @@ HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) if (obj || mNegativeCache.isPresent(hash) || !theApp || !theApp->getHashNodeLDB()) return obj; - std::string sData; + if (mEphemeralDB) + { + obj = LLRetrieve(hash, theApp->getEphemeralLDB()); + if (obj) + return obj; + } { LoadEvent::autoptr event(theApp->getJobQueue().getLoadEventAP(jtHO_READ, "HOS::retrieve")); - leveldb::Status st = theApp->getHashNodeLDB()->Get(leveldb::ReadOptions(), - leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), &sData); - if (!st.ok()) + obj = LLRetrieve(hash, theApp->getHashNodeLDB()); + if (!obj) { - assert(st.IsNotFound()); + mNegativeCache.add(hash); return obj; } } - const unsigned char* bufPtr = reinterpret_cast(&sData[0]); - uint32 index = htonl(*reinterpret_cast(bufPtr)); - int htype = bufPtr[8]; - - obj = boost::make_shared(static_cast(htype), index, - bufPtr + 9, sData.size() - 9, hash); mCache.canonicalize(hash, obj); + if (mEphemeralDB) + LLWrite(obj, theApp->getEphemeralLDB()); + WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db"; return obj; } @@ -196,6 +245,7 @@ bool HashedObjectStore::storeSQLite(HashedObjectType type, uint32 index, // else // WriteLog (lsTRACE, HashedObject) << "HOS: already had " << hash; mNegativeCache.del(hash); + return true; } @@ -223,6 +273,9 @@ void HashedObjectStore::bulkWriteSQLite(Job&) #ifndef NO_SQLITE3_PREPARE + if (mEphemeralDB) + LLWrite(set, theApp->getEphemeralLDB()); + { Database* db = theApp->getHashNodeDB()->getDB(); static SqliteStatement pStB(db->getSqliteDB(), "BEGIN TRANSACTION;", !theConfig.RUN_STANDALONE); @@ -308,6 +361,13 @@ HashedObject::pointer HashedObjectStore::retrieveSQLite(const uint256& hash) if (mNegativeCache.isPresent(hash)) return obj; + if (mEphemeralDB) + { + obj = LLRetrieve(hash, theApp->getEphemeralLDB()); + if (obj) + return obj; + } + if (!theApp || !theApp->getHashNodeDB()) return obj; @@ -387,6 +447,9 @@ HashedObject::pointer HashedObjectStore::retrieveSQLite(const uint256& hash) obj = boost::make_shared(htype, index, data, hash); mCache.canonicalize(hash, obj); + if (mEphemeralDB) + LLWrite(obj, theApp->getEphemeralLDB()); + WriteLog (lsTRACE, HashedObject) << "HOS: " << hash << " fetch: in db"; return obj; } diff --git a/src/cpp/ripple/HashedObject.h b/src/cpp/ripple/HashedObject.h index afc678284..25c05aeba 100644 --- a/src/cpp/ripple/HashedObject.h +++ b/src/cpp/ripple/HashedObject.h @@ -69,7 +69,7 @@ protected: std::vector< boost::shared_ptr > mWriteSet; bool mWritePending; - bool mLevelDB; + bool mLevelDB, mEphemeralDB; public: From 60ed891a968e59f0f8ca184be655d63a4f9e09f2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 13:24:20 -0700 Subject: [PATCH 13/27] Support ephemeral cache with SQLite. --- src/cpp/ripple/Application.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index a4bc761b9..466899ad8 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -179,18 +179,6 @@ void Application::setup() StopSustain(); exit(3); } - - if (!theConfig.LDB_EPHEMERAL.empty()) - { - leveldb::Status status = leveldb::DB::Open(options, theConfig.LDB_EPHEMERAL, &mEphemeralLDB); - if (!status.ok() || !mEphemeralLDB) - { - WriteLog(lsFATAL, Application) << "Unable to open/create epehemeral db: " - << theConfig.LDB_EPHEMERAL << " " << status.ToString(); - StopSustain(); - exit(3); - } - } } else { @@ -199,6 +187,18 @@ void Application::setup() t5.join(); } + if (!theConfig.LDB_EPHEMERAL.empty()) + { + leveldb::Status status = leveldb::DB::Open(options, theConfig.LDB_EPHEMERAL, &mEphemeralLDB); + if (!status.ok() || !mEphemeralLDB) + { + WriteLog(lsFATAL, Application) << "Unable to open/create epehemeral db: " + << theConfig.LDB_EPHEMERAL << " " << status.ToString(); + StopSustain(); + exit(3); + } + } + mTxnDB->getDB()->setupCheckpointing(&mJobQueue); mLedgerDB->getDB()->setupCheckpointing(&mJobQueue); From 72e0ac7cd5a4fedf91823d3eff8838e424b70350 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 13:25:34 -0700 Subject: [PATCH 14/27] Bugfix. --- src/cpp/ripple/Application.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 466899ad8..9304cb14d 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -160,16 +160,17 @@ void Application::setup() boost::thread t7(boost::bind(&InitDB, &mPathFindDB, "pathfind.db", PathFindDBInit, PathFindDBCount)); t4.join(); t6.join(); t7.join(); + leveldb::Options options; + options.create_if_missing = true; + options.block_cache = leveldb::NewLRUCache(theConfig.getSize(siHashNodeDBCache) * 1024 * 1024); + if (theConfig.NODE_SIZE >= 2) + options.filter_policy = leveldb::NewBloomFilterPolicy(10); + if (theConfig.LDB_IMPORT) + options.write_buffer_size = 32 << 20; + if (mHashedObjectStore.isLevelDB()) { WriteLog (lsINFO, Application) << "LevelDB used for nodes"; - leveldb::Options options; - options.create_if_missing = true; - options.block_cache = leveldb::NewLRUCache(theConfig.getSize(siHashNodeDBCache) * 1024 * 1024); - if (theConfig.NODE_SIZE >= 2) - options.filter_policy = leveldb::NewBloomFilterPolicy(10); - if (theConfig.LDB_IMPORT) - options.write_buffer_size = 32 << 20; leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { From 1875ddd506c2e3264079a32daad89d937c178cdc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 13:38:58 -0700 Subject: [PATCH 15/27] When an object is found in the epehermal DB, put it in the cache. --- src/cpp/ripple/HashedObject.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 096b9546c..f8044cf6b 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -192,7 +192,10 @@ HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) { obj = LLRetrieve(hash, theApp->getEphemeralLDB()); if (obj) + { + mCache.canonicalize(hash, obj); return obj; + } } { @@ -365,7 +368,10 @@ HashedObject::pointer HashedObjectStore::retrieveSQLite(const uint256& hash) { obj = LLRetrieve(hash, theApp->getEphemeralLDB()); if (obj) + { + mCache.canonicalize(hash, obj); return obj; + } } if (!theApp || !theApp->getHashNodeDB()) From 4492f0a1d2c91f288b7a01316d4797ddbec556ec Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 5 Jun 2013 16:39:55 -0700 Subject: [PATCH 16/27] server_info cleanup. --- src/cpp/ripple/NetworkOPs.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 5eac4efe3..eaeba324b 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1309,7 +1309,6 @@ Json::Value NetworkOPs::getServerInfo(bool human, bool admin) Json::Value l(Json::objectValue); l["seq"] = Json::UInt(lpClosed->getLedgerSeq()); l["hash"] = lpClosed->getHash().GetHex(); - l["validated"] = valid; if (!human) { l["base_fee"] = Json::Value::UInt(baseFee); @@ -1335,7 +1334,10 @@ Json::Value NetworkOPs::getServerInfo(bool human, bool admin) l["age"] = Json::UInt(age); } } - info["closed_ledger"] = l; + if (valid) + info["validated_ledger"] = l; + else + info["closed_ledger"] = l; } return info; From 7643f3673c18eee1ab43acf179a7b9efa2dff656 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 7 Jun 2013 05:07:56 -0700 Subject: [PATCH 17/27] Manipulations of remote_endpoint must be done in a try/catch block to avoid race conditions. --- src/cpp/ripple/Peer.cpp | 18 +++++++++++++++--- src/cpp/ripple/RPCDoor.cpp | 10 +++++++++- src/cpp/ripple/WSHandler.h | 8 +++++++- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index e3dcda098..be23d1375 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -278,9 +278,21 @@ void Peer::handleConnect(const boost::system::error_code& error, boost::asio::ip // - We don't bother remembering the inbound IP or port. Only useful for debugging. void Peer::connected(const boost::system::error_code& error) { - boost::asio::ip::tcp::endpoint ep = getSocket().remote_endpoint(); - int iPort = ep.port(); - std::string strIp = ep.address().to_string(); + boost::asio::ip::tcp::endpoint ep; + int iPort; + std::string strIp; + + try + { + ep = getSocket().remote_endpoint(); + iPort = ep.port(); + strIp = ep.address().to_string(); + } + catch (...) + { + detach("edc", false); + return; + } mClientConnect = false; mIpPortConnect = make_pair(strIp, iPort); diff --git a/src/cpp/ripple/RPCDoor.cpp b/src/cpp/ripple/RPCDoor.cpp index da79c9b48..214afe3c2 100644 --- a/src/cpp/ripple/RPCDoor.cpp +++ b/src/cpp/ripple/RPCDoor.cpp @@ -50,8 +50,16 @@ void RPCDoor::handleConnect(RPCServer::pointer new_connection, if (!error) { // Restrict callers by IP - if (!isClientAllowed(new_connection->getSocket().remote_endpoint().address().to_string())) + try { + if (!isClientAllowed(new_connection->getSocket().remote_endpoint().address().to_string())) + { + startListening(); + return; + } + } + catch (...) + { // client may have disconnected startListening(); return; } diff --git a/src/cpp/ripple/WSHandler.h b/src/cpp/ripple/WSHandler.h index 0b62ec3f2..4903d44a5 100644 --- a/src/cpp/ripple/WSHandler.h +++ b/src/cpp/ripple/WSHandler.h @@ -142,7 +142,13 @@ public: { boost::mutex::scoped_lock sl(mMapLock); - mMap[cpClient] = boost::make_shared< WSConnection >(this, cpClient); + try + { + mMap[cpClient] = boost::make_shared< WSConnection >(this, cpClient); + } + catch (...) + { + } } void on_pong(connection_ptr cpClient, std::string data) From ae6d06085c4270070688e3fc7a786e0638730704 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 19:23:47 -0700 Subject: [PATCH 18/27] Cleanup bit tests. --- src/cpp/ripple/Peer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index be23d1375..cbec95bc0 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -849,7 +849,9 @@ static void checkTransaction(Job&, int flags, SerializedTransaction::pointer stx #endif Transaction::pointer tx; - if ((flags & SF_SIGGOOD) != 0) + if (isSetBit(flags, SF_SIGGOOD)) + tx = boost::make_shared(stx, false); + else { tx = boost::make_shared(stx, true); if (tx->getStatus() == INVALID) @@ -861,10 +863,8 @@ static void checkTransaction(Job&, int flags, SerializedTransaction::pointer stx else theApp->getSuppression().setFlag(stx->getTransactionID(), SF_SIGGOOD); } - else - tx = boost::make_shared(stx, false); - theApp->getOPs().processTransaction(tx, (flags & SF_TRUSTED) != 0); + theApp->getOPs().processTransaction(tx, isSetBit(flags, SF_TRUSTED)); #ifndef TRUST_NETWORK } @@ -891,13 +891,13 @@ void Peer::recvTransaction(ripple::TMTransaction& packet, ScopedLock& MasterLock int flags; if (!theApp->isNew(stx->getTransactionID(), mPeerId, flags)) { // we have seen this transaction recently - if ((flags & SF_BAD) != 0) + if (isSetBit(flags, SF_BAD)) { punishPeer(LT_InvalidSignature); return; } - if ((flags & SF_RETRY) == 0) + if (!isSetBit(flags, SF_RETRY)) return; } WriteLog (lsDEBUG, Peer) << "Got new transaction from peer"; From a1b05bede76d3526a5fc4c7afa09d348b49271ae Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 10:54:31 -0700 Subject: [PATCH 19/27] Reduce logging. --- src/cpp/ripple/LedgerAcquire.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/LedgerAcquire.cpp b/src/cpp/ripple/LedgerAcquire.cpp index 3c528a5b1..d69911cac 100644 --- a/src/cpp/ripple/LedgerAcquire.cpp +++ b/src/cpp/ripple/LedgerAcquire.cpp @@ -621,7 +621,7 @@ void LedgerAcquire::filterNodes(std::vector& nodeIDs, std::vector Date: Mon, 10 Jun 2013 10:26:58 -0700 Subject: [PATCH 20/27] Query the LES, not the Ledger. This should fix the bug where OfferCreate doesn't take all crossing offers. --- src/cpp/ripple/OfferCreateTransactor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/OfferCreateTransactor.cpp b/src/cpp/ripple/OfferCreateTransactor.cpp index 821b33916..d227cb49c 100644 --- a/src/cpp/ripple/OfferCreateTransactor.cpp +++ b/src/cpp/ripple/OfferCreateTransactor.cpp @@ -136,7 +136,7 @@ TER OfferCreateTransactor::takeOffers( && saSubTakerPays.isPositive() && saSubTakerGets.isPositive()) { - sleOfferDir = mEngine->entryCache(ltDIR_NODE, mEngine->getLedger()->getNextLedgerIndex(uTipIndex, uBookEnd)); + sleOfferDir = mEngine->entryCache(ltDIR_NODE, lesActive.getNextLedgerIndex(uTipIndex, uBookEnd)); if (sleOfferDir) { uTipIndex = sleOfferDir->getIndex(); From 78eb8a0cdca53efd10e4313966e3bf6a2daadf14 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 19:29:24 -0700 Subject: [PATCH 21/27] Clean this up a bit more. --- src/cpp/ripple/Peer.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index cbec95bc0..6fd3deb13 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -852,17 +852,16 @@ static void checkTransaction(Job&, int flags, SerializedTransaction::pointer stx if (isSetBit(flags, SF_SIGGOOD)) tx = boost::make_shared(stx, false); else - { tx = boost::make_shared(stx, true); - if (tx->getStatus() == INVALID) - { - theApp->getSuppression().setFlag(stx->getTransactionID(), SF_BAD); - Peer::punishPeer(peer, LT_InvalidSignature); - return; - } - else - theApp->getSuppression().setFlag(stx->getTransactionID(), SF_SIGGOOD); + + if (tx->getStatus() == INVALID) + { + theApp->getSuppression().setFlag(stx->getTransactionID(), SF_BAD); + Peer::punishPeer(peer, LT_InvalidSignature); + return; } + else + theApp->getSuppression().setFlag(stx->getTransactionID(), SF_SIGGOOD); theApp->getOPs().processTransaction(tx, isSetBit(flags, SF_TRUSTED)); From e89efd567461afa2a9e8d2f3db1ec0f4490a1301 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 21:14:20 -0700 Subject: [PATCH 22/27] This index was never used. --- src/cpp/ripple/DBInit.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cpp/ripple/DBInit.cpp b/src/cpp/ripple/DBInit.cpp index da9978822..6f4459129 100644 --- a/src/cpp/ripple/DBInit.cpp +++ b/src/cpp/ripple/DBInit.cpp @@ -283,9 +283,6 @@ const char *HashNodeDBInit[] = { Object BLOB \ );", - "CREATE INDEX ObjectLocate ON \ - CommittedObjects(LedgerIndex, ObjType);", - "END TRANSACTION;" }; From ec04108910213b3935aebc4e3502868db69a8476 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 21:40:34 -0700 Subject: [PATCH 23/27] Correctly handle emptying an inner node. --- src/cpp/ripple/SHAMapNodes.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/SHAMapNodes.cpp b/src/cpp/ripple/SHAMapNodes.cpp index 2efd7ea3c..716fdeb52 100644 --- a/src/cpp/ripple/SHAMapNodes.cpp +++ b/src/cpp/ripple/SHAMapNodes.cpp @@ -343,6 +343,8 @@ bool SHAMapTreeNode::updateHash() assert(nh == s.getSHA512Half()); #endif } + else + nh.zero(); } else if (mType == tnTRANSACTION_NM) { From 58d7e62ad9f1b1e11491c35820a97801b2620163 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 21:53:50 -0700 Subject: [PATCH 24/27] Fullbelow cache improvements. Conflicts: src/cpp/ripple/ripple_SHAMap.h --- src/cpp/ripple/RPCHandler.cpp | 2 ++ src/cpp/ripple/SHAMap.h | 3 ++- src/cpp/ripple/SHAMapSync.cpp | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index 67bc87269..ae9bbe8d9 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -2297,6 +2297,8 @@ Json::Value RPCHandler::doGetCounts(Json::Value jvRequest, int& cost, ScopedLock ret["ledger_hit_rate"] = theApp->getLedgerMaster().getCacheHitRate(); ret["AL_hit_rate"] = AcceptedLedger::getCacheHitRate(); + ret["fullbelow_size"] = SHAMap::getFullBelowSize(); + std::string uptime; int s = UptimeTimer::getInstance().getElapsedSeconds(); textTime(uptime, s, "year", 365*24*60*60); diff --git a/src/cpp/ripple/SHAMap.h b/src/cpp/ripple/SHAMap.h index 2e3ea10e5..bc9bfdc4e 100644 --- a/src/cpp/ripple/SHAMap.h +++ b/src/cpp/ripple/SHAMap.h @@ -480,7 +480,8 @@ public: typedef std::pair< uint256, std::vector > fetchPackEntry_t; std::list getFetchPack(SHAMap* have, bool includeLeaves, int max); - static void sweep() { fullBelowCache.sweep(); } + static int getFullBelowCacheSize() { return fullBelowCache.getSize(); } + static void sweep() { fullBelowCache.sweep(); } }; #endif diff --git a/src/cpp/ripple/SHAMapSync.cpp b/src/cpp/ripple/SHAMapSync.cpp index e282609ea..2114cba97 100644 --- a/src/cpp/ripple/SHAMapSync.cpp +++ b/src/cpp/ripple/SHAMapSync.cpp @@ -11,7 +11,7 @@ static const uint256 uZero; -KeyCache SHAMap::fullBelowCache("fullBelowCache", 65536, 240); +KeyCache SHAMap::fullBelowCache("fullBelowCache", 524288, 240); void SHAMap::getMissingNodes(std::vector& nodeIDs, std::vector& hashes, int max, SHAMapSyncFilter* filter) From 67b92d4d87edec45d30cf1596c215cb07126cebc Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 22:00:09 -0700 Subject: [PATCH 25/27] Bugfix. --- modules/ripple_basics/containers/ripple_KeyCache.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/ripple_basics/containers/ripple_KeyCache.h b/modules/ripple_basics/containers/ripple_KeyCache.h index 64137d31c..622d5d1a3 100644 --- a/modules/ripple_basics/containers/ripple_KeyCache.h +++ b/modules/ripple_basics/containers/ripple_KeyCache.h @@ -28,19 +28,19 @@ public: assert((size >= 0) && (age > 2)); } - void getSize() + int getSize() { boost::mutex::scoped_lock sl(mNCLock); return mCache.size(); } - void getTargetSize() + int getTargetSize() { boost::mutex::scoped_lock sl(mNCLock); return mTargetSize; } - void getTargetAge() + int getTargetAge() { boost::mutex::scoped_lock sl(mNCLock); return mTargetAge; From e18122df775c6888b68bbba0593d94f11ee25eb2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 22:00:42 -0700 Subject: [PATCH 26/27] Get it right. --- src/cpp/ripple/SHAMap.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/SHAMap.h b/src/cpp/ripple/SHAMap.h index bc9bfdc4e..19953edfe 100644 --- a/src/cpp/ripple/SHAMap.h +++ b/src/cpp/ripple/SHAMap.h @@ -480,8 +480,8 @@ public: typedef std::pair< uint256, std::vector > fetchPackEntry_t; std::list getFetchPack(SHAMap* have, bool includeLeaves, int max); - static int getFullBelowCacheSize() { return fullBelowCache.getSize(); } - static void sweep() { fullBelowCache.sweep(); } + static int getFullBelowSize() { return fullBelowCache.getSize(); } + static void sweep() { fullBelowCache.sweep(); } }; #endif From de13b60dd43d5263c19d2dfa765e2a2e55ed9f20 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 10 Jun 2013 22:28:22 -0700 Subject: [PATCH 27/27] Spread fetch pack requests correctly. --- src/cpp/ripple/LedgerMaster.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 2b9fac9a1..8666a1d85 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -326,7 +326,7 @@ bool LedgerMaster::acquireMissingLedger(Ledger::ref origLedger, const uint256& l { if (count++ == 0) target = peer; - else if ((rand() % count) == 0) + else if ((rand() % ++count) == 0) target = peer; } }