From dada8eb7decb22b8ac77a08e30a2013679f03343 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 8 May 2013 11:38:30 -0700 Subject: [PATCH 01/40] LevelDB import speedups. --- src/cpp/ripple/Application.cpp | 2 ++ src/cpp/ripple/UpdateTables.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 5bec4cefc..49581d466 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -178,6 +178,8 @@ void Application::setup() options.block_cache = leveldb::NewLRUCache(theConfig.getSize(siHashNodeDBCache) * 1024 * 1024); if (theConfig.NODE_SIZE >= 2) options.filter_policy = leveldb::NewBloomFilterPolicy(10); + if (theConfig.LDB_IMPORT) + options.write_buffer_size = 32 << 20; leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { diff --git a/src/cpp/ripple/UpdateTables.cpp b/src/cpp/ripple/UpdateTables.cpp index 1fa0f9726..557e26b35 100644 --- a/src/cpp/ripple/UpdateTables.cpp +++ b/src/cpp/ripple/UpdateTables.cpp @@ -116,7 +116,7 @@ void Application::updateTables(bool ldbImport) if (theConfig.LDB_IMPORT) { Log(lsWARNING) << "Importing SQLite -> LevelDB"; - theApp->getHashedObjectStore().import(hashPath.string(), true); + theApp->getHashedObjectStore().import(hashPath.string(), false); Log(lsWARNING) << "Remove or remname the hashnode.db file"; } else From f6903508cf84178ebc96ae0c4abbd2ff8d22b00e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 8 May 2013 12:01:19 -0700 Subject: [PATCH 02/40] One more round of LeveDB import speedups. --- src/cpp/ripple/HashedObject.cpp | 41 ++++++++++++++++++--------------- src/cpp/ripple/HashedObject.h | 2 +- src/cpp/ripple/UpdateTables.cpp | 2 +- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index a12d03875..8a2eb4eed 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -366,12 +366,15 @@ HashedObject::pointer HashedObjectStore::retrieveSQLite(const uint256& hash) #ifdef USE_LEVELDB -int HashedObjectStore::import(const std::string& file, bool checkHashes) +int HashedObjectStore::import(const std::string& file) { cLog(lsWARNING) << "Hashed object import from \"" << file << "\"."; UPTR_T importDB(new SqliteDatabase(file.c_str())); importDB->connect(); + leveldb::DB* db = theApp->getHashNodeLDB(); + leveldb::WriteOptions wo; + int count = 0; SQL_FOREACH(importDB, "SELECT * FROM CommittedObjects;") @@ -386,17 +389,19 @@ int HashedObjectStore::import(const std::string& file, bool checkHashes) } else { - std::vector data; + std::vector rawData; + int size = importDB->getBinary("Object", NULL, 0); + rawData.resize(9 + size); + unsigned char* bufPtr = &rawData.front(); + + importDB->getBinary("Object", bufPtr + 9, size); + + uint32 index = importDB->getBigInt("LedgerIndex"); + *reinterpret_cast(bufPtr + 0) = ntohl(index); + *reinterpret_cast(bufPtr + 4) = ntohl(index); + std::string type; importDB->getStr("ObjType", type); - uint32 index = importDB->getBigInt("LedgerIndex"); - - int size = importDB->getBinary("Object", NULL, 0); - data.resize(size); - importDB->getBinary("Object", &(data.front()), size); - - assert(Serializer::getSHA512Half(data) == hash); - HashedObjectType htype = hotUNKNOWN; switch (type[0]) { @@ -408,17 +413,17 @@ int HashedObjectStore::import(const std::string& file, bool checkHashes) assert(false); cLog(lsERROR) << "Invalid hashed object"; } + *(bufPtr + 8) = static_cast(htype); - if (checkHashes && Serializer::getSHA512Half(data) != hash) + leveldb::Status st = db->Put(wo, + leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), + leveldb::Slice(reinterpret_cast(bufPtr), rawData.size())); + if (!st.ok()) { - cLog(lsWARNING) << "Hash mismatch in import table " << hash - << " " << Serializer::getSHA512Half(data); - } - else - { - store(htype, index, data, hash); - ++count; + cLog(lsFATAL) << "Failed to store hash node"; + assert(false); } + ++count; } if ((count % 10000) == 0) { diff --git a/src/cpp/ripple/HashedObject.h b/src/cpp/ripple/HashedObject.h index 1389108c0..e66f1f680 100644 --- a/src/cpp/ripple/HashedObject.h +++ b/src/cpp/ripple/HashedObject.h @@ -100,7 +100,7 @@ public: void tune(int size, int age); void sweep() { mCache.sweep(); mNegativeCache.sweep(); } - int import(const std::string& fileName, bool checkHashes); + int import(const std::string& fileName); }; #endif diff --git a/src/cpp/ripple/UpdateTables.cpp b/src/cpp/ripple/UpdateTables.cpp index 557e26b35..6db98d772 100644 --- a/src/cpp/ripple/UpdateTables.cpp +++ b/src/cpp/ripple/UpdateTables.cpp @@ -116,7 +116,7 @@ void Application::updateTables(bool ldbImport) if (theConfig.LDB_IMPORT) { Log(lsWARNING) << "Importing SQLite -> LevelDB"; - theApp->getHashedObjectStore().import(hashPath.string(), false); + theApp->getHashedObjectStore().import(hashPath.string()); Log(lsWARNING) << "Remove or remname the hashnode.db file"; } else From bf032b246807128aaaacf01d6c71763f58d41119 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 8 May 2013 17:23:18 -0700 Subject: [PATCH 03/40] Emergency fix. --- src/cpp/ripple/RippleCalc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cpp/ripple/RippleCalc.cpp b/src/cpp/ripple/RippleCalc.cpp index 9be60945a..362531572 100644 --- a/src/cpp/ripple/RippleCalc.cpp +++ b/src/cpp/ripple/RippleCalc.cpp @@ -2155,6 +2155,11 @@ TER RippleCalc::calcNodeAccountRev(const unsigned int uNode, PathState& psCur, c % psCur.saOutAct % psCur.saOutReq); + if (!saCurWantedReq.isPositive()) + { // TEMPORARY emergency fix + return tefEXCEPTION; + } + assert(saCurWantedReq.isPositive()); // Rate: quality in : 1.0 From a9b1c74df26ada161affea8f9c00fae239586764 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 9 May 2013 17:36:02 -0700 Subject: [PATCH 04/40] Avoid a crash scenario if we interrogate the remote endpoint after it disconnected. --- src/cpp/ripple/RPCServer.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/RPCServer.cpp b/src/cpp/ripple/RPCServer.cpp index f3b81e577..53044300e 100644 --- a/src/cpp/ripple/RPCServer.cpp +++ b/src/cpp/ripple/RPCServer.cpp @@ -146,7 +146,14 @@ std::string RPCServer::handleRequest(const std::string& requestStr) return HTTPReply(400, "params unparseable"); } - mRole = iAdminGet(jvRequest, mSocket.remote_endpoint().address().to_string()); + try + { + mRole = iAdminGet(jvRequest, mSocket.remote_endpoint().address().to_string()); + } + catch (...) + { // endpoint already disconnected + return ""; + } if (RPCHandler::FORBID == mRole) { From 614f29bf3698591afb2c1920ac58af579c856685 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 9 May 2013 12:28:43 -0700 Subject: [PATCH 05/40] Too many threads can cause an I/O bottleneck. --- src/cpp/ripple/JobQueue.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/JobQueue.cpp b/src/cpp/ripple/JobQueue.cpp index f6ce0bcf4..5429ba20c 100644 --- a/src/cpp/ripple/JobQueue.cpp +++ b/src/cpp/ripple/JobQueue.cpp @@ -239,6 +239,8 @@ void JobQueue::setThreadCount(int c) c = boost::thread::hardware_concurrency(); if (c < 0) c = 2; + if (c > 4) // I/O will bottleneck + c = 4; c += 2; cLog(lsINFO) << "Auto-tuning to " << c << " validation/transaction/proposal threads"; } From 51d7340364305b35ca50b09610c0a985cc39324f Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 14:41:11 -0700 Subject: [PATCH 06/40] Avoid the creation of large numbers of small LDB files. --- src/cpp/ripple/Application.cpp | 1 + src/cpp/ripple/Config.cpp | 3 ++- src/cpp/ripple/Config.h | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 49581d466..4fd2bbd11 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -180,6 +180,7 @@ void Application::setup() options.filter_policy = leveldb::NewBloomFilterPolicy(10); if (theConfig.LDB_IMPORT) options.write_buffer_size = 32 << 20; + options.write_buffer_size = std::max(options.write_buffer_size, theConfig.getSize(siLDBWriteSize) << 20); leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index 0d33383ad..15115dc42 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -515,7 +515,8 @@ int Config::getSize(SizedItemName item) { siLineCacheAge, { 500, 600, 1800, 3600, 7200 } }, { siHashNodeDBCache, { 24, 48, 64, 128, 256 } }, { siTxnDBCache, { 4, 12, 48, 96, 192 } }, - { siLgrDBCache, { 4, 8, 32, 64, 128 } } + { siLgrDBCache, { 4, 8, 32, 64, 128 } }, + { siLDBWriteSize, { 8, 16, 32, 64, 128 } } }; for (int i = 0; i < (sizeof(sizeTable) / sizeof(SizedItem)); ++i) diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index b662aec3d..d8b884c97 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -67,7 +67,8 @@ enum SizedItemName siLineCacheAge, siHashNodeDBCache, siTxnDBCache, - siLgrDBCache + siLgrDBCache, + siLDBWriteSize }; struct SizedItem From 28f4bef71a685ebd88c61ca6d2d87c92401543c1 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 16:39:17 -0700 Subject: [PATCH 07/40] Cleanup messages. --- src/cpp/ripple/Application.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 4fd2bbd11..fe84fb531 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -172,7 +172,7 @@ void Application::setup() #ifdef USE_LEVELDB if (mHashedObjectStore.isLevelDB()) { - cLog(lsFATAL) << "LDB"; + cLog(lsINFO) << "LevelDB used for nodes"; leveldb::Options options; options.create_if_missing = true; options.block_cache = leveldb::NewLRUCache(theConfig.getSize(siHashNodeDBCache) * 1024 * 1024); @@ -191,7 +191,7 @@ void Application::setup() else #endif { - cLog(lsFATAL) << "SQLite"; + cLog(lsINFO) << "SQLite used for nodes"; boost::thread t5(boost::bind(&InitDB, &mHashNodeDB, "hashnode.db", HashNodeDBInit, HashNodeDBCount)); t5.join(); } From 764e2efa81618673e6852209ed79aeea6bbd0b55 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 8 May 2013 15:28:52 -0700 Subject: [PATCH 08/40] Add 'ripple line to self' to list of internal errors we check metadata for. --- src/cpp/ripple/TransactionCheck.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/cpp/ripple/TransactionCheck.cpp b/src/cpp/ripple/TransactionCheck.cpp index 9e4355320..2d4a0f8ea 100644 --- a/src/cpp/ripple/TransactionCheck.cpp +++ b/src/cpp/ripple/TransactionCheck.cpp @@ -56,6 +56,16 @@ bool TransactionEngine::checkInvariants(TER result, const SerializedTransaction& } else if (entry.mAction == taaCREATE) { + if (entry.mEntry->getType() == ltRIPPLE_STATE) + { + if (entry.mEntry->getFieldAmount(sfLowLimit).getIssuer() == + entry.mEntry->getFieldAmount(sfHighLimit).getIssuer()) + { + cLog(lsFATAL) << "Ripple line to self"; + assert(false); + return tefINTERNAL; + } + } if (entry.mEntry->getType() == ltACCOUNT_ROOT) // account created xrpChange += entry.mEntry->getFieldAmount(sfBalance).getSNValue(); } From 4177f3459571413aa79287b63873dde3e878dc2e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 9 May 2013 21:59:01 -0700 Subject: [PATCH 09/40] Make sure to always set the result, not just the status. --- src/cpp/ripple/NetworkOPs.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 795748382..643239144 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -324,6 +324,7 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, if ((newFlags & SF_BAD) != 0) { // cached bad trans->setStatus(INVALID); + trans->setResult(temBAD_SIGNATURE); return trans; } @@ -333,6 +334,7 @@ Transaction::pointer NetworkOPs::processTransaction(Transaction::pointer trans, { cLog(lsINFO) << "Transaction has bad signature"; trans->setStatus(INVALID); + trans->setResult(temBAD_SIGNATURE); theApp->isNewFlag(trans->getID(), SF_BAD); return trans; } From 4fe89f21832181b6fc2fd451d001790e4653c7da Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 17:29:10 -0700 Subject: [PATCH 10/40] Fix a compilation bug. --- src/cpp/ripple/Application.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index fe84fb531..031c26136 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -180,7 +180,8 @@ void Application::setup() options.filter_policy = leveldb::NewBloomFilterPolicy(10); if (theConfig.LDB_IMPORT) options.write_buffer_size = 32 << 20; - options.write_buffer_size = std::max(options.write_buffer_size, theConfig.getSize(siLDBWriteSize) << 20); + options.write_buffer_size = std::max(options.write_buffer_size, + static_cast(theConfig.getSize(siLDBWriteSize) << 20)); leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { From c0ed34fb557d9ffa747cb667fa8f569c36e11adb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 17:29:19 -0700 Subject: [PATCH 11/40] Detect broken accttx databases. --- src/cpp/ripple/UpdateTables.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/cpp/ripple/UpdateTables.cpp b/src/cpp/ripple/UpdateTables.cpp index 6db98d772..5be595b2f 100644 --- a/src/cpp/ripple/UpdateTables.cpp +++ b/src/cpp/ripple/UpdateTables.cpp @@ -107,6 +107,13 @@ void Application::updateTables(bool ldbImport) assert(!schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "foobar")); addTxnSeqField(); + if (schemaHas(theApp->getTxnDB(), "AccountTransactions", 0, "PRIMARY")) + { + Log(lsFATAL) << "AccountTransactions database should not have a primary key"; + StopSustain(); + exit(1); + } + #ifdef USE_LEVELDB if (theApp->getHashedObjectStore().isLevelDB()) { @@ -123,6 +130,7 @@ void Application::updateTables(bool ldbImport) { Log(lsWARNING) << "SQLite hashnode database exists. Please either remove or import"; Log(lsWARNING) << "To import, start with the '--import' option. Otherwise, remove hashnode.db"; + StopSustain(); exit(1); } } From bf6f5c18647b48dc6703aaa7f2958cbd1448426e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 17:43:51 -0700 Subject: [PATCH 12/40] Don't use 'front' on a string. --- src/cpp/ripple/HashedObject.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 8a2eb4eed..c3cf13958 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -114,7 +114,7 @@ HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) return obj; } - const unsigned char* bufPtr = reinterpret_cast(&sData.front()); + const unsigned char* bufPtr = reinterpret_cast(&sData[0]); uint32 index = htonl(*reinterpret_cast(bufPtr)); int htype = bufPtr[8]; From acdb8969a374f1aec735a2ee91bc2d271c83fa51 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 18:52:46 -0700 Subject: [PATCH 13/40] Fix for C++ defect 106. --- src/cpp/ripple/OrderBookDB.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cpp/ripple/OrderBookDB.cpp b/src/cpp/ripple/OrderBookDB.cpp index 4ebac5749..9bfcd25ed 100644 --- a/src/cpp/ripple/OrderBookDB.cpp +++ b/src/cpp/ripple/OrderBookDB.cpp @@ -1,7 +1,9 @@ +#include "OrderBookDB.h" + #include +#include #include "Application.h" -#include "OrderBookDB.h" #include "Log.h" SETUP_LOG(); @@ -53,8 +55,10 @@ void OrderBookDB::setup(Ledger::ref ledger) OrderBook::pointer book = boost::make_shared(boost::cref(index), boost::cref(ci), boost::cref(co), boost::cref(ii), boost::cref(io)); - mSourceMap[currencyIssuer_ct(ci, ii)].push_back(book); - mDestMap[currencyIssuer_ct(co, io)].push_back(book); + mSourceMap[currencyIssuer_ct(boost::reference_wrapper(ci), + boost::reference_wrapper(ii))].push_back(book); + mDestMap[currencyIssuer_ct(boost::reference_wrapper(co), + boost::reference_wrapper(io))].push_back(book); } } From b68ad91bdc9ecae267f68c9ab3dcdd86918b4952 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 19:07:08 -0700 Subject: [PATCH 14/40] Workaround DR106. --- src/cpp/ripple/OrderBookDB.cpp | 6 ++---- src/cpp/ripple/OrderBookDB.h | 5 +++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/cpp/ripple/OrderBookDB.cpp b/src/cpp/ripple/OrderBookDB.cpp index 9bfcd25ed..0a7be2783 100644 --- a/src/cpp/ripple/OrderBookDB.cpp +++ b/src/cpp/ripple/OrderBookDB.cpp @@ -55,10 +55,8 @@ void OrderBookDB::setup(Ledger::ref ledger) OrderBook::pointer book = boost::make_shared(boost::cref(index), boost::cref(ci), boost::cref(co), boost::cref(ii), boost::cref(io)); - mSourceMap[currencyIssuer_ct(boost::reference_wrapper(ci), - boost::reference_wrapper(ii))].push_back(book); - mDestMap[currencyIssuer_ct(boost::reference_wrapper(co), - boost::reference_wrapper(io))].push_back(book); + mSourceMap[currencyIssuer_ct(ci, ii)].push_back(book); + mDestMap[currencyIssuer_ct(co, io)].push_back(book); } } diff --git a/src/cpp/ripple/OrderBookDB.h b/src/cpp/ripple/OrderBookDB.h index e90f37768..f7dbd0fa8 100644 --- a/src/cpp/ripple/OrderBookDB.h +++ b/src/cpp/ripple/OrderBookDB.h @@ -16,7 +16,12 @@ // typedef std::pair currencyIssuer_t; + +#ifdef C11X typedef std::pair currencyIssuer_ct; +#else +typedef std::pair currencyIssuer_ct; // C++ defect 106 +#endif class BookListeners { From 3009b9d54028684c2609b8474f72cee9ec13951a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 19:10:36 -0700 Subject: [PATCH 15/40] Another DR106 fix. --- src/cpp/ripple/Pathfinder.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cpp/ripple/Pathfinder.cpp b/src/cpp/ripple/Pathfinder.cpp index 7101f9458..0a33234c4 100644 --- a/src/cpp/ripple/Pathfinder.cpp +++ b/src/cpp/ripple/Pathfinder.cpp @@ -794,7 +794,11 @@ bool Pathfinder::matchesOrigin(const uint160& currency, const uint160& issuer) int Pathfinder::getPathsOut(const uint160& currencyID, const uint160& accountID, bool authRequired, bool isDstCurrency, const uint160& dstAccount) { +#ifdef C11X std::pair accountCurrency(currencyID, accountID); +#else + std::pair accountCurrency(currencyID, accountID); +#endif boost::unordered_map, int>::iterator it = mPOMap.find(accountCurrency); if (it != mPOMap.end()) return it->second; From 419f223f2e3ead8b887a7614151b79556f139a61 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 19:13:06 -0700 Subject: [PATCH 16/40] DR106 --- src/cpp/ripple/PathDB.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/cpp/ripple/PathDB.h b/src/cpp/ripple/PathDB.h index 59a4d5747..0a22843c1 100644 --- a/src/cpp/ripple/PathDB.h +++ b/src/cpp/ripple/PathDB.h @@ -7,7 +7,12 @@ #include "TaggedCache.h" typedef std::pair currencyIssuer_t; + +#ifdef C11X typedef std::pair currencyIssuer_ct; +#else +typedef std::pair currencyIssuer_ct; +#endif class PathDBEntry { From 0b563acea0e17202cd1a2d0ef5a844b78e68911b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 19:14:25 -0700 Subject: [PATCH 17/40] c++03 fix. --- src/cpp/ripple/PathDB.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/PathDB.h b/src/cpp/ripple/PathDB.h index 0a22843c1..0cc1effe1 100644 --- a/src/cpp/ripple/PathDB.h +++ b/src/cpp/ripple/PathDB.h @@ -69,11 +69,13 @@ public: const uint160& currencyTo, const uint160& issuerTo); }; +#ifdef C11X extern std::size_t hash_value(const currencyIssuer_ct& ci) { std::size_t r = hash_value(ci.second); return ci.first.hash_combine(r); } +#endif static inline std::size_t hash_value(const currencyIssuer_t& ci) { From 89018ad224e9fe4d14d160bed43cb5fb044f2270 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 11 May 2013 19:21:48 -0700 Subject: [PATCH 18/40] FreeBSD fix. --- SConstruct | 1 + 1 file changed, 1 insertion(+) diff --git a/SConstruct b/SConstruct index f35252552..6028302ff 100644 --- a/SConstruct +++ b/SConstruct @@ -94,6 +94,7 @@ if not FreeBSD: # Apparently, pkg-config --libs protobuf on bsd fails to provide this necessary include dir. if FreeBSD: env.Append(LINKFLAGS = ['-I/usr/local/include']) + env.Append(CXXFLAGS = ['-DOS_FREEBSD']) env.Append( LIBS = [ From 0193c48bf8de915b46a9c0ae90d805f0a9dac7f6 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 12 May 2013 17:27:03 -0700 Subject: [PATCH 19/40] Extra logging to try to track down why we're crashing. --- src/cpp/ripple/RippleCalc.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cpp/ripple/RippleCalc.cpp b/src/cpp/ripple/RippleCalc.cpp index 362531572..41254788e 100644 --- a/src/cpp/ripple/RippleCalc.cpp +++ b/src/cpp/ripple/RippleCalc.cpp @@ -3028,7 +3028,10 @@ int iPass = 0; BOOST_FOREACH(const uint256& uOfferIndex, vuUnfundedBecame) { if (tesSUCCESS == terResult) + { + cLog(lsDEBUG) << "Became unfunded " << uOfferIndex.GetHex(); terResult = lesActive.offerDelete(uOfferIndex); + } } } @@ -3036,7 +3039,10 @@ int iPass = 0; BOOST_FOREACH(const uint256& uOfferIndex, rc.musUnfundedFound) { if (tesSUCCESS == terResult) + { + cLog(lsDEBUG) << "Delete unfunded " << uOfferIndex.GetHex(); terResult = lesActive.offerDelete(uOfferIndex); + } } } From 40355c7b58055f5d2ffb4f8763526edbc9d1ff4e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 12 May 2013 21:33:08 -0700 Subject: [PATCH 20/40] More predictable LevelDB latency, LevelDB write batching. --- src/cpp/ripple/Application.cpp | 2 - src/cpp/ripple/Config.cpp | 1 - src/cpp/ripple/Config.h | 1 - src/cpp/ripple/HashedObject.cpp | 82 ++++++++++++++++++++++++--------- src/cpp/ripple/HashedObject.h | 4 +- 5 files changed, 63 insertions(+), 27 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 031c26136..ea7f1b66a 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -180,8 +180,6 @@ void Application::setup() options.filter_policy = leveldb::NewBloomFilterPolicy(10); if (theConfig.LDB_IMPORT) options.write_buffer_size = 32 << 20; - options.write_buffer_size = std::max(options.write_buffer_size, - static_cast(theConfig.getSize(siLDBWriteSize) << 20)); leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index 15115dc42..891bf45fc 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -516,7 +516,6 @@ int Config::getSize(SizedItemName item) { siHashNodeDBCache, { 24, 48, 64, 128, 256 } }, { siTxnDBCache, { 4, 12, 48, 96, 192 } }, { siLgrDBCache, { 4, 8, 32, 64, 128 } }, - { siLDBWriteSize, { 8, 16, 32, 64, 128 } } }; for (int i = 0; i < (sizeof(sizeTable) / sizeof(SizedItem)); ++i) diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index d8b884c97..bbb3a6521 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -68,7 +68,6 @@ enum SizedItemName siHashNodeDBCache, siTxnDBCache, siLgrDBCache, - siLDBWriteSize }; struct SizedItem diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index c3cf13958..20bff54f8 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -1,8 +1,8 @@ - #include "HashedObject.h" #ifdef USE_LEVELDB #include "leveldb/db.h" +#include "leveldb/write_batch.h" #endif #include @@ -68,34 +68,72 @@ bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, HashedObject::pointer object = boost::make_shared(type, index, data, hash); if (!mCache.canonicalize(hash, object)) { - LoadEvent::autoptr event(theApp->getJobQueue().getLoadEventAP(jtHO_WRITE, "HOS::store")); - - std::vector rawData(9 + data.size()); - unsigned char* bufPtr = &rawData.front(); - - *reinterpret_cast(bufPtr + 0) = ntohl(index); - *reinterpret_cast(bufPtr + 4) = ntohl(index); - *(bufPtr + 8) = static_cast(type); - memcpy(bufPtr + 9, &data.front(), data.size()); - - leveldb::Status st = theApp->getHashNodeLDB()->Put(leveldb::WriteOptions(), - leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), - leveldb::Slice(reinterpret_cast(bufPtr), 9 + data.size())); - if (!st.ok()) + boost::mutex::scoped_lock sl(mWriteMutex); + mWriteSet.push_back(object); + if (!mWritePending) { - cLog(lsFATAL) << "Failed to store hash node"; - assert(false); + mWritePending = true; + theApp->getJobQueue().addJob(jtWRITE, "HashedObject::store", + BIND_TYPE(&HashedObjectStore::bulkWriteLevelDB, this)); } } - else - cLog(lsDEBUG) << "HOS: store race"; + mNegativeCache.del(hash); return true; } +void HashedObjectStore::bulkWriteLevelDB() +{ + assert(mLevelDB); + while (1) + { + std::vector< boost::shared_ptr > set; + set.reserve(128); + + { + boost::mutex::scoped_lock sl(mWriteMutex); + mWriteSet.swap(set); + assert(mWriteSet.empty()); + ++mWriteGeneration; + mWriteCondition.notify_all(); + if (set.empty()) + { + mWritePending = false; + return; + } + } + + { + leveldb::WriteBatch batch; + + BOOST_FOREACH(const boost::shared_ptr& it, set) + { + const HashedObject& obj = *it; + std::vector rawData(9 + obj.mData.size()); + unsigned char* bufPtr = &rawData.front(); + + *reinterpret_cast(bufPtr + 0) = ntohl(obj.mLedgerIndex); + *reinterpret_cast(bufPtr + 4) = ntohl(obj.mLedgerIndex); + *(bufPtr + 8) = static_cast(obj.mType); + memcpy(bufPtr + 9, &obj.mData.front(), obj.mData.size()); + + batch.Put(leveldb::Slice(reinterpret_cast(obj.mHash.begin()), obj.mHash.size()), + leveldb::Slice(reinterpret_cast(bufPtr), rawData.size())); + } + + leveldb::Status st = theApp->getHashNodeLDB()->Write(leveldb::WriteOptions(), &batch); + if (!st.ok()) + { + cLog(lsFATAL) << "Failed to store hash node"; + assert(false); + } + } + } +} + HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) { HashedObject::pointer obj = mCache.fetch(hash); - if (obj) + if (obj || mNegativeCache.isPresent(hash)) return obj; if (!theApp || !theApp->getHashNodeLDB()) @@ -153,7 +191,7 @@ bool HashedObjectStore::storeSQLite(HashedObjectType type, uint32 index, { mWritePending = true; theApp->getJobQueue().addJob(jtWRITE, "HashedObject::store", - BIND_TYPE(&HashedObjectStore::bulkWrite, this)); + BIND_TYPE(&HashedObjectStore::bulkWriteSQLite, this)); } } // else @@ -172,7 +210,7 @@ void HashedObjectStore::waitWrite() mWriteCondition.wait(sl); } -void HashedObjectStore::bulkWrite() +void HashedObjectStore::bulkWriteSQLite() { assert(!mLevelDB); while (1) diff --git a/src/cpp/ripple/HashedObject.h b/src/cpp/ripple/HashedObject.h index e66f1f680..014977ef5 100644 --- a/src/cpp/ripple/HashedObject.h +++ b/src/cpp/ripple/HashedObject.h @@ -88,14 +88,16 @@ public: bool storeSQLite(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); HashedObject::pointer retrieveSQLite(const uint256& hash); + void bulkWriteSQLite(); #ifdef USE_LEVELDB bool storeLevelDB(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash); HashedObject::pointer retrieveLevelDB(const uint256& hash); + void bulkWriteLevelDB(); #endif - void bulkWrite(); + void waitWrite(); void tune(int size, int age); void sweep() { mCache.sweep(); mNegativeCache.sweep(); } From c54c832382d9b9f7a826be6f661063b1f3fa1c1a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sun, 12 May 2013 21:43:57 -0700 Subject: [PATCH 21/40] Cleanups. --- src/cpp/ripple/HashedObject.cpp | 51 ++++++++++++++------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 20bff54f8..92f21172b 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -48,22 +48,28 @@ void HashedObjectStore::tune(int size, int age) mCache.setTargetAge(age); } +void HashedObjectStore::waitWrite() +{ + boost::mutex::scoped_lock sl(mWriteMutex); + int gen = mWriteGeneration; + while (mWritePending && (mWriteGeneration == gen)) + mWriteCondition.wait(sl); +} + #ifdef USE_LEVELDB bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, const std::vector& data, const uint256& hash) { // return: false = already in cache, true = added to cache if (!theApp->getHashNodeLDB()) - { - cLog(lsWARNING) << "HOS: no db"; return true; - } + if (mCache.touch(hash)) - { - cLog(lsTRACE) << "HOS: " << hash << " store: incache"; return false; - } + +#ifdef PARANOID assert(hash == Serializer::getSHA512Half(data)); +#endif HashedObject::pointer object = boost::make_shared(type, index, data, hash); if (!mCache.canonicalize(hash, object)) @@ -133,23 +139,20 @@ void HashedObjectStore::bulkWriteLevelDB() HashedObject::pointer HashedObjectStore::retrieveLevelDB(const uint256& hash) { HashedObject::pointer obj = mCache.fetch(hash); - if (obj || mNegativeCache.isPresent(hash)) + if (obj || mNegativeCache.isPresent(hash) || !theApp || !theApp->getHashNodeLDB()) return obj; - if (!theApp || !theApp->getHashNodeLDB()) - { - cLog(lsWARNING) << "HOS: no db"; - return obj; - } - - LoadEvent::autoptr event(theApp->getJobQueue().getLoadEventAP(jtHO_READ, "HOS::retrieve")); std::string sData; - leveldb::Status st = theApp->getHashNodeLDB()->Get(leveldb::ReadOptions(), - leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), &sData); - if (!st.ok()) + { - assert(st.IsNotFound()); - return obj; + LoadEvent::autoptr event(theApp->getJobQueue().getLoadEventAP(jtHO_READ, "HOS::retrieve")); + leveldb::Status st = theApp->getHashNodeLDB()->Get(leveldb::ReadOptions(), + leveldb::Slice(reinterpret_cast(hash.begin()), hash.size()), &sData); + if (!st.ok()) + { + assert(st.IsNotFound()); + return obj; + } } const unsigned char* bufPtr = reinterpret_cast(&sData[0]); @@ -200,16 +203,6 @@ bool HashedObjectStore::storeSQLite(HashedObjectType type, uint32 index, return true; } -void HashedObjectStore::waitWrite() -{ - if (mLevelDB) - return; - boost::mutex::scoped_lock sl(mWriteMutex); - int gen = mWriteGeneration; - while (mWritePending && (mWriteGeneration == gen)) - mWriteCondition.wait(sl); -} - void HashedObjectStore::bulkWriteSQLite() { assert(!mLevelDB); From cd08616cd270706c4631ce389aa0dee150ac3235 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 13 May 2013 06:45:49 -0700 Subject: [PATCH 22/40] Report more details if we can't open/create the LDB database. --- src/cpp/ripple/Application.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index ea7f1b66a..9840ffc9d 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -183,7 +183,8 @@ void Application::setup() leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { - cLog(lsFATAL) << "Unable to open/create hash node db"; + cLog(lsFATAL) << "Unable to open/create hash node db: " << (theConfig.DATA_DIR / "hashnode").string(); + StopSustain(); exit(3); } } From 312ed4ec7f7dcc9ef51349b850f016dca9623682 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 13 May 2013 06:52:37 -0700 Subject: [PATCH 23/40] Output the leveldb error message if we can't open/create the DB. --- src/cpp/ripple/Application.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 9840ffc9d..4c6cd9334 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -183,7 +183,9 @@ void Application::setup() leveldb::Status status = leveldb::DB::Open(options, (theConfig.DATA_DIR / "hashnode").string(), &mHashNodeLDB); if (!status.ok() || !mHashNodeLDB) { - cLog(lsFATAL) << "Unable to open/create hash node db: " << (theConfig.DATA_DIR / "hashnode").string(); + cLog(lsFATAL) << "Unable to open/create hash node db: " + << (theConfig.DATA_DIR / "hashnode").string() + << " " << status.ToString(); StopSustain(); exit(3); } From 36746f07b45d9b4d5488c05bd67c182eb62e620a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Mon, 13 May 2013 18:27:24 -0700 Subject: [PATCH 24/40] Bugfix. --- src/cpp/ripple/TransactionCheck.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/TransactionCheck.cpp b/src/cpp/ripple/TransactionCheck.cpp index 2d4a0f8ea..3ef97b9dd 100644 --- a/src/cpp/ripple/TransactionCheck.cpp +++ b/src/cpp/ripple/TransactionCheck.cpp @@ -9,7 +9,7 @@ SETUP_LOG(); bool TransactionEngine::checkInvariants(TER result, const SerializedTransaction& txn, TransactionEngineParams params) { #if 0 - const RippleAddress& srcAccount = txn.getFieldAccount(sfAccount); + RippleAddress srcAccount = txn.getFieldAccount(sfAccount); uint32 txnSeq = txn.getFieldU32(sfSequence); LedgerEntryAction leaAction; From 4f6344bb138521f649a456d45a5a9e33e53883a4 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Mon, 13 May 2013 22:07:46 -0700 Subject: [PATCH 25/40] Merge branch 'fixrequiredst' into master --- src/cpp/ripple/AccountSetTransactor.cpp | 4 +- test/account_set-test.js | 221 ++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 test/account_set-test.js diff --git a/src/cpp/ripple/AccountSetTransactor.cpp b/src/cpp/ripple/AccountSetTransactor.cpp index 31856044b..60bea27f4 100644 --- a/src/cpp/ripple/AccountSetTransactor.cpp +++ b/src/cpp/ripple/AccountSetTransactor.cpp @@ -32,7 +32,7 @@ TER AccountSetTransactor::doApply() if ((uTxFlags & tfRequireAuth) && !isSetBit(uFlagsIn, lsfRequireAuth)) { - if (mTxn.getFieldU32(sfOwnerCount)) + if (mTxnAccount->getFieldU32(sfOwnerCount)) { cLog(lsINFO) << "AccountSet: Retry: OwnerCount not zero."; @@ -62,7 +62,7 @@ TER AccountSetTransactor::doApply() return temINVALID_FLAG; } - if ((uTxFlags & tfOptionalDestTag) && !isSetBit(uFlagsIn, lsfRequireDestTag)) + if ((uTxFlags & tfRequireDestTag) && !isSetBit(uFlagsIn, lsfRequireDestTag)) { cLog(lsINFO) << "AccountSet: Set lsfRequireDestTag."; diff --git a/test/account_set-test.js b/test/account_set-test.js new file mode 100644 index 000000000..65352c352 --- /dev/null +++ b/test/account_set-test.js @@ -0,0 +1,221 @@ +var async = require("async"); +var buster = require("buster"); + +var Amount = require("ripple-lib").Amount; +var Remote = require("ripple-lib").Remote; +var Request = require("ripple-lib").Request; +var Server = require("./server").Server; + +var testutils = require("./testutils"); + +var config = require('ripple-lib').config.load(require('./config')); + +// How long to wait for server to start. +var serverDelay = 1500; + +buster.testRunner.timeout = 5000; + +buster.testCase("AccountSet", { + 'setUp' : testutils.build_setup(), + // 'setUp' : testutils.build_setup({verbose: true , no_server: false}), + 'tearDown' : testutils.build_teardown(), + + "RequireDestTag" : + function (done) { + var self = this; + + async.waterfall([ + function (callback) { + self.what = "Set RequireDestTag."; + + self.remote.transaction() + .account_set("root") + .set_flags('RequireDestTag') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check RequireDestTag"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !(m.node.Flags & Remote.flags.account_root.RequireDestTag); + + if (wrong) + console.log("Set RequireDestTag: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + function (callback) { + self.what = "Clear RequireDestTag."; + + self.remote.transaction() + .account_set("root") + .set_flags('OptionalDestTag') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check No RequireDestTag"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !!(m.node.Flags & Remote.flags.account_root.RequireDestTag); + + if (wrong) + console.log("Clear RequireDestTag: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + ], function (error) { + buster.refute(error, self.what); + done(); + }); + }, + + "RequireAuth" : + function (done) { + var self = this; + + async.waterfall([ + function (callback) { + self.what = "Set RequireAuth."; + + self.remote.transaction() + .account_set("root") + .set_flags('RequireAuth') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check RequireAuth"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !(m.node.Flags & Remote.flags.account_root.RequireAuth); + + if (wrong) + console.log("Set RequireAuth: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + function (callback) { + self.what = "Clear RequireAuth."; + + self.remote.transaction() + .account_set("root") + .set_flags('OptionalAuth') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check No RequireAuth"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !!(m.node.Flags & Remote.flags.account_root.RequireAuth); + + if (wrong) + console.log("Clear RequireAuth: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + // XXX Also check fails if something is owned. + ], function (error) { + buster.refute(error, self.what); + done(); + }); + }, + + "DisallowXRP" : + function (done) { + var self = this; + + async.waterfall([ + function (callback) { + self.what = "Set DisallowXRP."; + + self.remote.transaction() + .account_set("root") + .set_flags('DisallowXRP') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check DisallowXRP"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !(m.node.Flags & Remote.flags.account_root.DisallowXRP); + + if (wrong) + console.log("Set RequireDestTag: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + function (callback) { + self.what = "Clear DisallowXRP."; + + self.remote.transaction() + .account_set("root") + .set_flags('AllowXRP') + .on('proposed', function (m) { + //console.log("proposed: %s", JSON.stringify(m)); + + callback(m.result !== 'tesSUCCESS'); + }) + .submit(); + }, + function (callback) { + self.what = "Check AllowXRP"; + + self.remote.request_account_flags('root', 'CURRENT') + .on('success', function (m) { + var wrong = !!(m.node.Flags & Remote.flags.account_root.DisallowXRP); + + if (wrong) + console.log("Clear DisallowXRP: failed: %s", JSON.stringify(m)); + + callback(wrong); + }) + .request(); + }, + ], function (error) { + buster.refute(error, self.what); + done(); + }); + }, + +}); + +// vim:sw=2:sts=2:ts=8:et From d0d071c46b18537eeadef63c56d3b3195f074e17 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 8 May 2013 15:21:22 -0700 Subject: [PATCH 26/40] Detect server deadlocks and trigger the auto-restart mechanism. --- src/cpp/ripple/LoadManager.cpp | 24 +++++++++++++++++++++++- src/cpp/ripple/LoadManager.h | 3 +++ src/cpp/ripple/NetworkOPs.cpp | 2 ++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/LoadManager.cpp b/src/cpp/ripple/LoadManager.cpp index d83d87822..46965ff2b 100644 --- a/src/cpp/ripple/LoadManager.cpp +++ b/src/cpp/ripple/LoadManager.cpp @@ -22,7 +22,7 @@ int upTime() LoadManager::LoadManager(int creditRate, int creditLimit, int debitWarn, int debitLimit) : mCreditRate(creditRate), mCreditLimit(creditLimit), mDebitWarn(debitWarn), mDebitLimit(debitLimit), - mShutdown(false), mUptime(0), mCosts(LT_MAX) + mShutdown(false), mUptime(0), mDeadLock(0), mCosts(LT_MAX) { addLoadCost(LoadCost(LT_InvalidRequest, -10, LC_CPU | LC_Network)); addLoadCost(LoadCost(LT_RequestNoReply, -1, LC_CPU | LC_Disk)); @@ -67,6 +67,11 @@ LoadManager::~LoadManager() while (1); } +void LoadManager::noDeadLock() +{ + boost::mutex::scoped_lock sl(mLock); + mDeadLock = mUptime; +} int LoadManager::getCreditRate() const { @@ -321,6 +326,11 @@ int LoadManager::getUptime() return mUptime; } +static void LogDeadLock(int dlTime) +{ + cLog(lsWARNING) << "Server stalled for " << dlTime << " seconds."; +} + void LoadManager::threadEntry() { NameThread("loadmgr"); @@ -335,6 +345,18 @@ void LoadManager::threadEntry() return; } ++mUptime; + + int dlTime = mUptime - mDeadLock; + if (dlTime >= 10) + { + if ((dlTime % 10) == 0) + { + boost::thread(BIND_TYPE(&LogDeadLock, dlTime)).detach(); + } + + assert (dlTime < 180); + } + } bool change; diff --git a/src/cpp/ripple/LoadManager.h b/src/cpp/ripple/LoadManager.h index 392620241..07722b9e6 100644 --- a/src/cpp/ripple/LoadManager.h +++ b/src/cpp/ripple/LoadManager.h @@ -106,6 +106,8 @@ protected: int mUptime; int mSpace2[4]; + int mDeadLock; // Detect server deadlocks + mutable boost::mutex mLock; void canonicalize(LoadSource&, int upTime) const; @@ -141,6 +143,7 @@ public: int getCost(LoadType t) { return mCosts[static_cast(t)].mCost; } int getUptime(); + void noDeadLock(); }; class LoadFeeTrack diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 643239144..ca76eb9b3 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -595,6 +595,8 @@ void NetworkOPs::checkState(const boost::system::error_code& result) { ScopedLock sl(theApp->getMasterLock()); + theApp->getLoadManager().noDeadLock(); + std::vector peerList = theApp->getConnectionPool().getPeerVector(); // do we have sufficient peers? If not, we are disconnected. From d1c6b97643e7523d33436bbf46876bad02fc8176 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 14 May 2013 10:31:26 -0700 Subject: [PATCH 27/40] Report the number of "backed up" node writes in get_counts. --- src/cpp/ripple/HashedObject.cpp | 13 ++++++++++++- src/cpp/ripple/HashedObject.h | 2 ++ src/cpp/ripple/RPCHandler.cpp | 2 ++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/HashedObject.cpp b/src/cpp/ripple/HashedObject.cpp index 92f21172b..5ff173307 100644 --- a/src/cpp/ripple/HashedObject.cpp +++ b/src/cpp/ripple/HashedObject.cpp @@ -19,7 +19,7 @@ DECLARE_INSTANCE(HashedObject); HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) : mCache("HashedObjectStore", cacheSize, cacheAge), mNegativeCache("HashedObjectNegativeCache", 0, 120), - mWriteGeneration(0), mWritePending(false), mLevelDB(false) + mWriteGeneration(0), mWriteLoad(0), mWritePending(false), mLevelDB(false) { mWriteSet.reserve(128); @@ -56,6 +56,12 @@ void HashedObjectStore::waitWrite() mWriteCondition.wait(sl); } +int HashedObjectStore::getWriteLoad() +{ + boost::mutex::scoped_lock sl(mWriteMutex); + return std::max(mWriteLoad, static_cast(mWriteSet.size())); +} + #ifdef USE_LEVELDB bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, @@ -90,6 +96,7 @@ bool HashedObjectStore::storeLevelDB(HashedObjectType type, uint32 index, void HashedObjectStore::bulkWriteLevelDB() { assert(mLevelDB); + int setSize = 0; while (1) { std::vector< boost::shared_ptr > set; @@ -97,6 +104,7 @@ void HashedObjectStore::bulkWriteLevelDB() { boost::mutex::scoped_lock sl(mWriteMutex); + mWriteSet.swap(set); assert(mWriteSet.empty()); ++mWriteGeneration; @@ -104,8 +112,11 @@ void HashedObjectStore::bulkWriteLevelDB() if (set.empty()) { mWritePending = false; + mWriteLoad = 0; return; } + mWriteLoad = std::max(setSize, static_cast(mWriteSet.size())); + setSize = set.size(); } { diff --git a/src/cpp/ripple/HashedObject.h b/src/cpp/ripple/HashedObject.h index 014977ef5..cfc712f13 100644 --- a/src/cpp/ripple/HashedObject.h +++ b/src/cpp/ripple/HashedObject.h @@ -55,6 +55,7 @@ protected: boost::mutex mWriteMutex; boost::condition_variable mWriteCondition; int mWriteGeneration; + int mWriteLoad; std::vector< boost::shared_ptr > mWriteSet; bool mWritePending; @@ -101,6 +102,7 @@ public: void waitWrite(); void tune(int size, int age); void sweep() { mCache.sweep(); mNegativeCache.sweep(); } + int getWriteLoad(); int import(const std::string& fileName); }; diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index f3aa3aee9..ea616d940 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -2170,6 +2170,8 @@ Json::Value RPCHandler::doGetCounts(Json::Value jvRequest, int& cost, ScopedLock if (dbKB > 0) ret["dbKBTransaction"] = dbKB; + ret["write_load"] = theApp->getHashedObjectStore().getWriteLoad(); + std::string uptime; int s = upTime(); textTime(uptime, s, "year", 365*24*60*60); From 2a82aaa9ef33b3a9a7c8bae0fc237663ac934219 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 14 May 2013 11:00:41 -0700 Subject: [PATCH 28/40] Don't log a transaction receipt for each peer we get it from. --- src/cpp/ripple/Peer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index 98c7cbd38..8474db6de 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -866,7 +866,6 @@ static void checkTransaction(Job&, int flags, SerializedTransaction::pointer stx void Peer::recvTransaction(ripple::TMTransaction& packet) { - cLog(lsDEBUG) << "Got transaction from peer"; Transaction::pointer tx; #ifndef TRUST_NETWORK @@ -890,6 +889,8 @@ void Peer::recvTransaction(ripple::TMTransaction& packet) if ((flags & SF_RETRY) == 0) return; } + cLog(lsDEBUG) << "Got new transaction from peer"; + if (mCluster) flags |= SF_TRUSTED | SF_SIGGOOD; From bc30aeaa04a32a65d698125e31b21aadf5c8f1db Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 14 May 2013 11:01:52 -0700 Subject: [PATCH 29/40] Reduce logging. --- src/cpp/ripple/LedgerAcquire.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cpp/ripple/LedgerAcquire.cpp b/src/cpp/ripple/LedgerAcquire.cpp index 087db9fba..291ff8f8f 100644 --- a/src/cpp/ripple/LedgerAcquire.cpp +++ b/src/cpp/ripple/LedgerAcquire.cpp @@ -160,11 +160,11 @@ bool LedgerAcquire::tryLocal() { TransactionStateSF filter(mLedger->getLedgerSeq()); mLedger->peekTransactionMap()->fetchRoot(mLedger->getTransHash(), &filter); - cLog(lsDEBUG) << "Got root txn map locally"; + cLog(lsTRACE) << "Got root txn map locally"; std::vector h = mLedger->getNeededTransactionHashes(1, &filter); if (h.empty()) { - cLog(lsDEBUG) << "Had full txn map locally"; + cLog(lsTRACE) << "Had full txn map locally"; mHaveTransactions = true; } } @@ -187,11 +187,11 @@ bool LedgerAcquire::tryLocal() { AccountStateSF filter(mLedger->getLedgerSeq()); mLedger->peekAccountStateMap()->fetchRoot(mLedger->getAccountHash(), &filter); - cLog(lsDEBUG) << "Got root AS map locally"; + cLog(lsTRACE) << "Got root AS map locally"; std::vector h = mLedger->getNeededAccountStateHashes(1, &filter); if (h.empty()) { - cLog(lsDEBUG) << "Had full AS map locally"; + cLog(lsTRACE) << "Had full AS map locally"; mHaveState = true; } } From 43c3d8969cb20b1e761ddb8d8a99ec87ce99a9dd Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 14 May 2013 11:01:58 -0700 Subject: [PATCH 30/40] Prevent crashing when we try to delete a non-existent offer. --- src/cpp/ripple/LedgerEntrySet.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/LedgerEntrySet.cpp b/src/cpp/ripple/LedgerEntrySet.cpp index 9c51bcaa6..728242d77 100644 --- a/src/cpp/ripple/LedgerEntrySet.cpp +++ b/src/cpp/ripple/LedgerEntrySet.cpp @@ -985,8 +985,10 @@ TER LedgerEntrySet::offerDelete(SLE::ref sleOffer, const uint256& uOfferIndex, c TER LedgerEntrySet::offerDelete(const uint256& uOfferIndex) { SLE::pointer sleOffer = entryCache(ltOFFER, uOfferIndex); - const uint160 uOwnerID = sleOffer->getFieldAccount(sfAccount).getAccountID(); + if (!sleOffer) + return tesSUCCESS; + const uint160 uOwnerID = sleOffer->getFieldAccount160(sfAccount); return offerDelete(sleOffer, uOfferIndex, uOwnerID); } From 3eae3f45592463c59a30782af31d9ddc1750d726 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 14 May 2013 11:04:32 -0700 Subject: [PATCH 31/40] Log reduction. --- src/cpp/ripple/Ledger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index caef05f5c..6a37031d7 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -526,7 +526,7 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus) if (theApp->getJobQueue().getJobCountTotal(jtPUBOLDLEDGER) < 2) theApp->getLedgerMaster().resumeAcquiring(); else - cLog(lsDEBUG) << "no resume, too many pending ledger saves"; + cLog(lsTRACE) << "no resume, too many pending ledger saves"; } #ifndef NO_SQLITE3_PREPARE From 8d23509f5bc049759244a3e825cbb7f7c019d7fe Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:53:49 -0700 Subject: [PATCH 32/40] Operation to extract transaction ID from CanonicalTXSet entries. --- src/cpp/ripple/CanonicalTXSet.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/CanonicalTXSet.h b/src/cpp/ripple/CanonicalTXSet.h index fead540db..a28b004e5 100644 --- a/src/cpp/ripple/CanonicalTXSet.h +++ b/src/cpp/ripple/CanonicalTXSet.h @@ -23,6 +23,8 @@ public: bool operator==(const CanonicalTXKey& k) const { return mTXid == k.mTXid; } bool operator!=(const CanonicalTXKey& k) const { return mTXid != k.mTXid; } + + const uint256& getTXID() const { return mTXid; } }; class CanonicalTXSet From 7c0fc8c15548083d1cbbfd13cdfc58a3e7ff0a43 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Wed, 15 May 2013 21:07:01 +0000 Subject: [PATCH 33/40] Normalize line endings --- rippled.1 | Bin 3121 -> 3043 bytes src/cpp/json/LICENSE | Bin 67 -> 66 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/rippled.1 b/rippled.1 index c94248ae6baa9f963da886168721cd17b2df630f..8617000b89c97f6bc8e02bacaa3ca09c7bb5f982 100644 GIT binary patch delta 614 zcmZvY!E4iS6vt_rwArufJlM_+Cr?{cv`c9QDnlxjr8*dGu~Kx1;g{vtEs&-)$<(74 z5%IF_tB42v1H9<&9z2N${{le}JP4k=?q=H9%FxsM@;=}9-j}y~^!v*+pKJ+1 zRX6A@4ep#DJl6-GP7Hn)FrBDmnAQkrnfv!04{M29cj}NJNd{MIarCdTYl|8_jwi1#PiY{4EBbQ*ZfA#I zqv37i9vStv*{)tU%jl}da^?RaJwo$%svgH1wT?aY+J({GCa2O?;Fp3;?u$~#afRve zu3)v5Rl_Wk{WtO0j?7f?NX+$pd>j)$rb>t62v=NMupPbulg*&QXk2qKHe9B)HmPM8 xR9$T~2=7>9C)2Dk?WRwh%2cG0w1#q|)+dUS{-kZCtbAp~@h#Ob0WWtCk_vk}TO4$l97_I`HIQ zpz}NHMFc@Sc=6&)T}Nw+;R?GtOtlOJjtZxkI7f z?FOD(X8f&iADw-n0lb$w58O*#2EIw%1pZ0Q04GEvGw5Udffx^ZCQbw2iLw2gIITd` zL$brI*}o+jO$CaV$!_ZcD*q0+n!62w-=GhxRox$ z{WKjXJ_Kct6jz)WiAe4 zr}(E#3G7EkI~rZFX*005lI*2`jwV-}#PM%*C#xdxRkl$;W3xo@!Hmabn2lCLUu#tv zujM9-)ni317N_I`%?P-in-GZ@-_4l_{gS%{%*&Ifn6k9IVzjE{j&1|J^88TI{#UR% zI^WAz#tyP?1XOW-EXU6(1xS6FfQS6d;SoUait2_z`byoxQ3Az+^Ru#QLR!avrz(}> hhXwA$JfRySYLhmsIwNY++^9N-;5+#R1V7K;{R_fi!xjJl diff --git a/src/cpp/json/LICENSE b/src/cpp/json/LICENSE index d20fb29a7e23828c15487438c93ae790d2045707..1c37b1181031ce5347aa15666b380884e819d6d8 100644 GIT binary patch delta 6 NcmZ>En&8OD1po!{0b&3E delta 7 OcmZ>Ap5VyD%LM=jWdU#i From 2b39f6a618d36f41587585d133d769e1d8c376a6 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:54:30 -0700 Subject: [PATCH 34/40] Cache signature state inside SerializedTransaction. --- src/cpp/ripple/SerializedTransaction.cpp | 25 +++++++++++++++++++----- src/cpp/ripple/SerializedTransaction.h | 6 ++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/SerializedTransaction.cpp b/src/cpp/ripple/SerializedTransaction.cpp index 0d7dd34f2..ccd5f073e 100644 --- a/src/cpp/ripple/SerializedTransaction.cpp +++ b/src/cpp/ripple/SerializedTransaction.cpp @@ -11,7 +11,8 @@ SETUP_LOG(); DECLARE_INSTANCE(SerializedTransaction); -SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sfTransaction), mType(type) +SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sfTransaction), mType(type), + mSigGood(false), mSigBad(false) { mFormat = TransactionFormat::getTxnFormat(type); if (mFormat == NULL) @@ -23,7 +24,8 @@ SerializedTransaction::SerializedTransaction(TransactionType type) : STObject(sf setFieldU16(sfTransactionType, mFormat->t_type); } -SerializedTransaction::SerializedTransaction(const STObject& object) : STObject(object) +SerializedTransaction::SerializedTransaction(const STObject& object) : STObject(object), + mSigGood(false), mSigBad(false) { mType = static_cast(getFieldU16(sfTransactionType)); mFormat = TransactionFormat::getTxnFormat(mType); @@ -38,7 +40,8 @@ SerializedTransaction::SerializedTransaction(const STObject& object) : STObject( } } -SerializedTransaction::SerializedTransaction(SerializerIterator& sit) : STObject(sfTransaction) +SerializedTransaction::SerializedTransaction(SerializerIterator& sit) : STObject(sfTransaction), + mSigGood(false), mSigBad(false) { int length = sit.getBytesLeft(); if ((length < TransactionMinLen) || (length > TransactionMaxLen)) @@ -156,16 +159,28 @@ void SerializedTransaction::sign(const RippleAddress& naAccountPrivate) bool SerializedTransaction::checkSign() const { + if (mSigGood) + return true; + + if (mSigBad) + return false; + try { RippleAddress n; n.setAccountPublic(getFieldVL(sfSigningPubKey)); - return checkSign(n); + if (checkSign(n)) + { + mSigGood = true; + return true; + } } catch (...) { - return false; + ; } + mSigBad = true; + return false; } bool SerializedTransaction::checkSign(const RippleAddress& naAccountPublic) const diff --git a/src/cpp/ripple/SerializedTransaction.h b/src/cpp/ripple/SerializedTransaction.h index 5dfc24f27..6b9dde135 100644 --- a/src/cpp/ripple/SerializedTransaction.h +++ b/src/cpp/ripple/SerializedTransaction.h @@ -32,6 +32,8 @@ protected: SerializedTransaction* duplicate() const { return new SerializedTransaction(*this); } + mutable bool mSigGood, mSigBad; + public: SerializedTransaction(SerializerIterator& sit); SerializedTransaction(TransactionType type); @@ -69,6 +71,10 @@ public: void sign(const RippleAddress& naAccountPrivate); bool checkSign(const RippleAddress& naAccountPublic) const; bool checkSign() const; + bool isKnownGood() const { return mSigGood; } + bool isKnownBad() const { return mSigBad; } + void setGood() const { mSigGood = true; } + void setBad() const { mSigBad = true; } // SQL Functions static std::string getSQLValueHeader(); From e94802bc2a00b64a85891104d56d765ae414a37d Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:54:52 -0700 Subject: [PATCH 35/40] Don't stash the key when we don't need it. --- src/cpp/ripple/NetworkOPs.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 3ec5d8a93..4b09dde55 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -184,8 +184,7 @@ void NetworkOPs::submitTransaction(Job&, SerializedTransaction::pointer iTrans, { try { - RippleAddress fromPubKey = RippleAddress::createAccountPublic(trans->getSigningPubKey()); - if (!trans->checkSign(fromPubKey)) + if (!trans->checkSign()) { cLog(lsWARNING) << "Submitted transaction has bad signature"; theApp->isNewFlag(suppress, SF_BAD); From 28a88b57ac46cc0fd7d64f72812ce0c3285b8067 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:55:23 -0700 Subject: [PATCH 36/40] Bypass signature check if it has already been done. --- src/cpp/ripple/LedgerConsensus.cpp | 2 ++ src/cpp/ripple/LedgerMaster.cpp | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index a312d30ff..556ea0de9 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -1043,6 +1043,8 @@ int LedgerConsensus::applyTransaction(TransactionEngine& engine, SerializedTrans TransactionEngineParams parms = openLedger ? tapOPEN_LEDGER : tapNONE; if (retryAssured) parms = static_cast(parms | tapRETRY); + if (theApp->isNewFlag(txn->getTransactionID(), SF_SIGGOOD)) + parms = static_cast(parms | tapNO_CHECK_SIGN); cLog(lsDEBUG) << "TXN " << txn->getTransactionID() << (openLedger ? " open" : " closed") diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 23d2c7ab7..dd93c393e 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -101,8 +101,13 @@ Ledger::pointer LedgerMaster::closeLedger(bool recover) { try { + TransactionEngineParams tepFlags = tapOPEN_LEDGER; + + if (theApp->isNew(it->first.getTXID(), SF_SIGGOOD)); + tepFlags = static_cast(tepFlags | tapNO_CHECK_SIGN); + bool didApply; - mEngine.applyTransaction(*it->second, tapOPEN_LEDGER, didApply); + mEngine.applyTransaction(*it->second, tepFlags, didApply); if (didApply) ++recovers; } From 4d1b45906db6d5f629c902159c4335114a22ca1e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 15:55:35 -0700 Subject: [PATCH 37/40] Update the signature check flags if needed. --- src/cpp/ripple/Transactor.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/Transactor.cpp b/src/cpp/ripple/Transactor.cpp index 82b8ab9bc..daa6171b3 100644 --- a/src/cpp/ripple/Transactor.cpp +++ b/src/cpp/ripple/Transactor.cpp @@ -160,7 +160,6 @@ TER Transactor::checkSeq() // check stuff before you bother to lock the ledger TER Transactor::preCheck() { - mTxnAccountID = mTxn.getSourceAccount().getAccountID(); if (!mTxnAccountID) { @@ -177,11 +176,15 @@ TER Transactor::preCheck() mSigningPubKey = RippleAddress::createAccountPublic(mTxn.getSigningPubKey()); // Consistency: really signed. - if ( !isSetBit(mParams, tapNO_CHECK_SIGN) && !mTxn.checkSign(mSigningPubKey)) + if (!mTxn.isKnownGood()) { - cLog(lsWARNING) << "applyTransaction: Invalid transaction: bad signature"; - - return temINVALID; + if (mTxn.isKnownBad() || (!isSetBit(mParams, tapNO_CHECK_SIGN) && !mTxn.checkSign(mSigningPubKey))) + { + mTxn.setGood(); + cLog(lsWARNING) << "applyTransaction: Invalid transaction: bad signature"; + return temINVALID; + } + mTxn.isKnownGood(); } return tesSUCCESS; From 8ee6d2381f583760c618f3890a27af0f091713de Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 17:32:44 -0700 Subject: [PATCH 38/40] Upgrade LevelDB to 1.10.0, mostly for better write stall logging. --- src/cpp/leveldb/db/db_impl.cc | 17 +++++++++++---- src/cpp/leveldb/db/db_test.cc | 31 ++++++++++++++++++++++++++++ src/cpp/leveldb/db/dbformat.cc | 2 +- src/cpp/leveldb/include/leveldb/db.h | 2 +- src/cpp/leveldb/table/block.cc | 13 ++++++------ src/cpp/leveldb/table/table.cc | 1 - src/cpp/leveldb/table/table_test.cc | 30 +++++++++++++++++++++++++++ src/cpp/leveldb/util/cache.cc | 5 +---- src/cpp/leveldb/util/env_posix.cc | 4 ++-- src/cpp/leveldb/util/hash.cc | 11 ++++++++-- 10 files changed, 95 insertions(+), 21 deletions(-) diff --git a/src/cpp/leveldb/db/db_impl.cc b/src/cpp/leveldb/db/db_impl.cc index c9de169f2..cd765c523 100644 --- a/src/cpp/leveldb/db/db_impl.cc +++ b/src/cpp/leveldb/db/db_impl.cc @@ -310,16 +310,24 @@ Status DBImpl::Recover(VersionEdit* edit) { if (!s.ok()) { return s; } + std::set expected; + versions_->AddLiveFiles(&expected); uint64_t number; FileType type; std::vector logs; for (size_t i = 0; i < filenames.size(); i++) { - if (ParseFileName(filenames[i], &number, &type) - && type == kLogFile - && ((number >= min_log) || (number == prev_log))) { + if (ParseFileName(filenames[i], &number, &type)) { + expected.erase(number); + if (type == kLogFile && ((number >= min_log) || (number == prev_log))) logs.push_back(number); } } + if (!expected.empty()) { + char buf[50]; + snprintf(buf, sizeof(buf), "%d missing files; e.g.", + static_cast(expected.size())); + return Status::Corruption(buf, TableFileName(dbname_, *(expected.begin()))); + } // Recover in the order in which the logs were generated std::sort(logs.begin(), logs.end()); @@ -1268,10 +1276,11 @@ Status DBImpl::MakeRoomForWrite(bool force) { } else if (imm_ != NULL) { // We have filled up the current memtable, but the previous // one is still being compacted, so we wait. + Log(options_.info_log, "Current memtable full; waiting...\n"); bg_cv_.Wait(); } else if (versions_->NumLevelFiles(0) >= config::kL0_StopWritesTrigger) { // There are too many level-0 files. - Log(options_.info_log, "waiting...\n"); + Log(options_.info_log, "Too many L0 files; waiting...\n"); bg_cv_.Wait(); } else { // Attempt to switch to a new memtable and trigger compaction of old diff --git a/src/cpp/leveldb/db/db_test.cc b/src/cpp/leveldb/db/db_test.cc index 684ea3bdb..2f51296be 100644 --- a/src/cpp/leveldb/db/db_test.cc +++ b/src/cpp/leveldb/db/db_test.cc @@ -461,6 +461,20 @@ class DBTest { } return result; } + + bool DeleteAnSSTFile() { + std::vector filenames; + ASSERT_OK(env_->GetChildren(dbname_, &filenames)); + uint64_t number; + FileType type; + for (size_t i = 0; i < filenames.size(); i++) { + if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) { + ASSERT_OK(env_->DeleteFile(TableFileName(dbname_, number))); + return true; + } + } + return false; + } }; TEST(DBTest, Empty) { @@ -1567,6 +1581,23 @@ TEST(DBTest, ManifestWriteError) { } } +TEST(DBTest, MissingSSTFile) { + ASSERT_OK(Put("foo", "bar")); + ASSERT_EQ("bar", Get("foo")); + + // Dump the memtable to disk. + dbfull()->TEST_CompactMemTable(); + ASSERT_EQ("bar", Get("foo")); + + ASSERT_TRUE(DeleteAnSSTFile()); + Options options = CurrentOptions(); + options.paranoid_checks = true; + Status s = TryReopen(&options); + ASSERT_TRUE(!s.ok()); + ASSERT_TRUE(s.ToString().find("issing") != std::string::npos) + << s.ToString(); +} + TEST(DBTest, FilesDeletedAfterCompaction) { ASSERT_OK(Put("foo", "v2")); Compact("a", "z"); diff --git a/src/cpp/leveldb/db/dbformat.cc b/src/cpp/leveldb/db/dbformat.cc index 28e11b398..20a7ca446 100644 --- a/src/cpp/leveldb/db/dbformat.cc +++ b/src/cpp/leveldb/db/dbformat.cc @@ -26,7 +26,7 @@ std::string ParsedInternalKey::DebugString() const { (unsigned long long) sequence, int(type)); std::string result = "'"; - result += user_key.ToString(); + result += EscapeString(user_key.ToString()); result += buf; return result; } diff --git a/src/cpp/leveldb/include/leveldb/db.h b/src/cpp/leveldb/include/leveldb/db.h index 29d367447..a37c097d1 100644 --- a/src/cpp/leveldb/include/leveldb/db.h +++ b/src/cpp/leveldb/include/leveldb/db.h @@ -14,7 +14,7 @@ namespace leveldb { // Update Makefile if you change these static const int kMajorVersion = 1; -static const int kMinorVersion = 9; +static const int kMinorVersion = 10; struct Options; struct ReadOptions; diff --git a/src/cpp/leveldb/table/block.cc b/src/cpp/leveldb/table/block.cc index ab83c1112..79ea9d9ee 100644 --- a/src/cpp/leveldb/table/block.cc +++ b/src/cpp/leveldb/table/block.cc @@ -16,7 +16,7 @@ namespace leveldb { inline uint32_t Block::NumRestarts() const { - assert(size_ >= 2*sizeof(uint32_t)); + assert(size_ >= sizeof(uint32_t)); return DecodeFixed32(data_ + size_ - sizeof(uint32_t)); } @@ -27,11 +27,12 @@ Block::Block(const BlockContents& contents) if (size_ < sizeof(uint32_t)) { size_ = 0; // Error marker } else { - restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); - if (restart_offset_ > size_ - sizeof(uint32_t)) { - // The size is too small for NumRestarts() and therefore - // restart_offset_ wrapped around. + size_t max_restarts_allowed = (size_-sizeof(uint32_t)) / sizeof(uint32_t); + if (NumRestarts() > max_restarts_allowed) { + // The size is too small for NumRestarts() size_ = 0; + } else { + restart_offset_ = size_ - (1 + NumRestarts()) * sizeof(uint32_t); } } } @@ -253,7 +254,7 @@ class Block::Iter : public Iterator { }; Iterator* Block::NewIterator(const Comparator* cmp) { - if (size_ < 2*sizeof(uint32_t)) { + if (size_ < sizeof(uint32_t)) { return NewErrorIterator(Status::Corruption("bad block contents")); } const uint32_t num_restarts = NumRestarts(); diff --git a/src/cpp/leveldb/table/table.cc b/src/cpp/leveldb/table/table.cc index dbd6d3a1b..71c1756e5 100644 --- a/src/cpp/leveldb/table/table.cc +++ b/src/cpp/leveldb/table/table.cc @@ -228,7 +228,6 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k, !filter->KeyMayMatch(handle.offset(), k)) { // Not found } else { - Slice handle = iiter->value(); Iterator* block_iter = BlockReader(this, options, iiter->value()); block_iter->Seek(k); if (block_iter->Valid()) { diff --git a/src/cpp/leveldb/table/table_test.cc b/src/cpp/leveldb/table/table_test.cc index 57cea2533..c723bf84c 100644 --- a/src/cpp/leveldb/table/table_test.cc +++ b/src/cpp/leveldb/table/table_test.cc @@ -644,6 +644,36 @@ class Harness { Constructor* constructor_; }; +// Test empty table/block. +TEST(Harness, Empty) { + for (int i = 0; i < kNumTestArgs; i++) { + Init(kTestArgList[i]); + Random rnd(test::RandomSeed() + 1); + Test(&rnd); + } +} + +// Special test for a block with no restart entries. The C++ leveldb +// code never generates such blocks, but the Java version of leveldb +// seems to. +TEST(Harness, ZeroRestartPointsInBlock) { + char data[sizeof(uint32_t)]; + memset(data, 0, sizeof(data)); + BlockContents contents; + contents.data = Slice(data, sizeof(data)); + contents.cachable = false; + contents.heap_allocated = false; + Block block(contents); + Iterator* iter = block.NewIterator(BytewiseComparator()); + iter->SeekToFirst(); + ASSERT_TRUE(!iter->Valid()); + iter->SeekToLast(); + ASSERT_TRUE(!iter->Valid()); + iter->Seek("foo"); + ASSERT_TRUE(!iter->Valid()); + delete iter; +} + // Test the empty key TEST(Harness, SimpleEmptyKey) { for (int i = 0; i < kNumTestArgs; i++) { diff --git a/src/cpp/leveldb/util/cache.cc b/src/cpp/leveldb/util/cache.cc index 24f1f63f4..8b197bc02 100644 --- a/src/cpp/leveldb/util/cache.cc +++ b/src/cpp/leveldb/util/cache.cc @@ -116,7 +116,6 @@ class HandleTable { LRUHandle* h = list_[i]; while (h != NULL) { LRUHandle* next = h->next_hash; - Slice key = h->key(); uint32_t hash = h->hash; LRUHandle** ptr = &new_list[hash & (new_length - 1)]; h->next_hash = *ptr; @@ -160,7 +159,6 @@ class LRUCache { // mutex_ protects the following state. port::Mutex mutex_; size_t usage_; - uint64_t last_id_; // Dummy head of LRU list. // lru.prev is newest entry, lru.next is oldest entry. @@ -170,8 +168,7 @@ class LRUCache { }; LRUCache::LRUCache() - : usage_(0), - last_id_(0) { + : usage_(0) { // Make empty circular linked list lru_.next = &lru_; lru_.prev = &lru_; diff --git a/src/cpp/leveldb/util/env_posix.cc b/src/cpp/leveldb/util/env_posix.cc index db81f56d1..b328183ca 100644 --- a/src/cpp/leveldb/util/env_posix.cc +++ b/src/cpp/leveldb/util/env_posix.cc @@ -386,7 +386,7 @@ class PosixEnv : public Env { PosixEnv(); virtual ~PosixEnv() { fprintf(stderr, "Destroying Env::Default()\n"); - exit(1); + abort(); } virtual Status NewSequentialFile(const std::string& fname, @@ -589,7 +589,7 @@ class PosixEnv : public Env { void PthreadCall(const char* label, int result) { if (result != 0) { fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); - exit(1); + abort(); } } diff --git a/src/cpp/leveldb/util/hash.cc b/src/cpp/leveldb/util/hash.cc index ba1818082..07cf02206 100644 --- a/src/cpp/leveldb/util/hash.cc +++ b/src/cpp/leveldb/util/hash.cc @@ -6,6 +6,13 @@ #include "util/coding.h" #include "util/hash.h" +// The FALLTHROUGH_INTENDED macro can be used to annotate implicit fall-through +// between switch labels. The real definition should be provided externally. +// This one is a fallback version for unsupported compilers. +#ifndef FALLTHROUGH_INTENDED +#define FALLTHROUGH_INTENDED do { } while (0) +#endif + namespace leveldb { uint32_t Hash(const char* data, size_t n, uint32_t seed) { @@ -28,10 +35,10 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) { switch (limit - data) { case 3: h += data[2] << 16; - // fall through + FALLTHROUGH_INTENDED; case 2: h += data[1] << 8; - // fall through + FALLTHROUGH_INTENDED; case 1: h += data[0]; h *= m; From 1f3314f8f78ed3f767d252d61e1d4819f5ffa83b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 15 May 2013 20:26:12 -0700 Subject: [PATCH 39/40] Allow servers to explcitly vouch for correct signatures on transactions, proposals, and validations. This will allow cluster members to more safely distribute crypto work. --- src/cpp/ripple/ripple.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cpp/ripple/ripple.proto b/src/cpp/ripple/ripple.proto index 8e5def253..848802bdd 100644 --- a/src/cpp/ripple/ripple.proto +++ b/src/cpp/ripple/ripple.proto @@ -94,6 +94,7 @@ message TMTransaction { required bytes rawTransaction = 1; required TransactionStatus status = 2; optional uint64 receiveTimestamp = 3; + optional bool checkedSignature = 4; // no vouches for signature being correct } @@ -132,6 +133,7 @@ message TMProposeSet { required uint32 closeTime = 4; required bytes signature = 5; // signature of above fields optional bytes previousledger = 6; + optional bool checkedSignature = 7; // node vouches signature is correct repeated bytes addedTransactions = 10; // not required if number is large repeated bytes removedTransactions = 11; // not required if number is large } @@ -151,6 +153,7 @@ message TMHaveTransactionSet { // Used to sign a final closed ledger after reprocessing message TMValidation { required bytes validation = 1; // in SerializedValidation signed form + optional bool checkedSignature = 2; // node vouches signature is correct } From 68510178c23e16c77e2a582d47ffb136d3b263e5 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Thu, 23 May 2013 16:01:15 -0700 Subject: [PATCH 40/40] Add support for strict to CLI for account_info. --- src/cpp/ripple/CallRPC.cpp | 18 ++++++++++++++++-- src/cpp/ripple/main.cpp | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/cpp/ripple/CallRPC.cpp b/src/cpp/ripple/CallRPC.cpp index 78fc2579c..a347f7a63 100644 --- a/src/cpp/ripple/CallRPC.cpp +++ b/src/cpp/ripple/CallRPC.cpp @@ -437,7 +437,18 @@ Json::Value RPCParser::parseAccountLines(const Json::Value& jvParams) Json::Value RPCParser::parseAccountRaw(const Json::Value& jvParams, bool bPeer) { std::string strIdent = jvParams[0u].asString(); - std::string strPeer = bPeer && jvParams.size() >= 2 ? jvParams[1u].asString() : ""; + unsigned int iCursor = jvParams.size(); + bool bStrict = false; + std::string strPeer; + + if (!bPeer && iCursor >= 2 && jvParams[iCursor-1] == "strict") + { + bStrict = true; + --iCursor; + } + + if (bPeer && iCursor >= 2) + strPeer = jvParams[iCursor].asString(); int iIndex = 0; // int iIndex = jvParams.size() >= 2 ? lexical_cast_s(jvParams[1u].asString()) : 0; @@ -452,6 +463,9 @@ Json::Value RPCParser::parseAccountRaw(const Json::Value& jvParams, bool bPeer) jvRequest["account"] = strIdent; + if (bStrict) + jvRequest["strict"] = 1; + if (iIndex) jvRequest["account_index"] = iIndex; @@ -465,7 +479,7 @@ Json::Value RPCParser::parseAccountRaw(const Json::Value& jvParams, bool bPeer) jvRequest["peer"] = strPeer; } - if (jvParams.size() == (2+bPeer) && !jvParseLedger(jvRequest, jvParams[1u+bPeer].asString())) + if (iCursor == (2+bPeer) && !jvParseLedger(jvRequest, jvParams[1u+bPeer].asString())) return rpcError(rpcLGR_IDX_MALFORMED); return jvRequest; diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index 15a23767d..7cfb5cb0b 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -70,7 +70,7 @@ void printHelp(const po::options_description& desc) cerr << desc << endl; cerr << "Commands: " << endl; - cerr << " account_info |||| []" << endl; + cerr << " account_info |||| [] [strict]" << endl; cerr << " account_lines |\"\" []" << endl; cerr << " account_offers || []" << endl; cerr << " account_tx accountID [ledger_min [ledger_max [limit [offset]]]] [binary] [count] [descending]" << endl;