From 5b0d456aa409d28d174edfc8c775d4cf804defad Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Jun 2013 15:23:38 -0700 Subject: [PATCH 1/8] Add lsfSell flag to indicate an offer was placed as a sell. --- modules/ripple_data/protocol/ripple_LedgerFormat.h | 1 + src/cpp/ripple/OfferCreateTransactor.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/modules/ripple_data/protocol/ripple_LedgerFormat.h b/modules/ripple_data/protocol/ripple_LedgerFormat.h index 13e3d8280..02af945dc 100644 --- a/modules/ripple_data/protocol/ripple_LedgerFormat.h +++ b/modules/ripple_data/protocol/ripple_LedgerFormat.h @@ -44,6 +44,7 @@ enum LedgerSpecificFlags // ltOFFER lsfPassive = 0x00010000, + lsfSell = 0x00020000, // True, offer was placed as a sell. // ltRIPPLE_STATE lsfLowReserve = 0x00010000, // True, if entry counts toward reserve. diff --git a/src/cpp/ripple/OfferCreateTransactor.cpp b/src/cpp/ripple/OfferCreateTransactor.cpp index 3e7112c48..6a7bdfba2 100644 --- a/src/cpp/ripple/OfferCreateTransactor.cpp +++ b/src/cpp/ripple/OfferCreateTransactor.cpp @@ -640,6 +640,9 @@ TER OfferCreateTransactor::doApply() if (bPassive) sleOffer->setFlag(lsfPassive); + if (bSell) + sleOffer->setFlag(lsfSell); + WriteLog (lsINFO, OfferCreateTransactor) << boost::str(boost::format("OfferCreate: final terResult=%s sleOffer=%s") % transToken(terResult) % sleOffer->getJson(0)); From 0c46180bd11d21c4097f64e20d9805f2f6a4e746 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 13 Jun 2013 21:08:02 -0700 Subject: [PATCH 2/8] Report the write load for SQLite too. --- src/cpp/ripple/ripple_HashedObjectStore.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cpp/ripple/ripple_HashedObjectStore.cpp b/src/cpp/ripple/ripple_HashedObjectStore.cpp index fbdc18614..eb32e01a1 100644 --- a/src/cpp/ripple/ripple_HashedObjectStore.cpp +++ b/src/cpp/ripple/ripple_HashedObjectStore.cpp @@ -243,6 +243,7 @@ bool HashedObjectStore::storeSQLite(HashedObjectType type, uint32 index, void HashedObjectStore::bulkWriteSQLite(Job&) { assert(!mLevelDB); + int setSize = 0; while (1) { std::vector< boost::shared_ptr > set; @@ -257,8 +258,11 @@ void HashedObjectStore::bulkWriteSQLite(Job&) if (set.empty()) { mWritePending = false; + mWriteLoad = 0; return; } + mWriteLoad = std::max(setSize, static_cast(mWriteSet.size())); + setSize = set.size(); } // WriteLog (lsTRACE, HashedObject) << "HOS: writing " << set.size(); From 80520cbce79d86a812c873e7df81faa2e4a17db2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 14 Jun 2013 10:58:47 -0700 Subject: [PATCH 3/8] LevelDB issue 178 fix: cannot resize a level 0 compaction set --- src/cpp/leveldb/db/version_set.cc | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/cpp/leveldb/db/version_set.cc b/src/cpp/leveldb/db/version_set.cc index 7d0a5de2b..4fd1ddef2 100644 --- a/src/cpp/leveldb/db/version_set.cc +++ b/src/cpp/leveldb/db/version_set.cc @@ -1331,14 +1331,19 @@ Compaction* VersionSet::CompactRange( } // Avoid compacting too much in one shot in case the range is large. - const uint64_t limit = MaxFileSizeForLevel(level); - uint64_t total = 0; - for (size_t i = 0; i < inputs.size(); i++) { - uint64_t s = inputs[i]->file_size; - total += s; - if (total >= limit) { - inputs.resize(i + 1); - break; + // But we cannot do this for level-0 since level-0 files can overlap + // and we must not pick one file and drop another older file if the + // two files overlap. + if (level > 0) { + const uint64_t limit = MaxFileSizeForLevel(level); + uint64_t total = 0; + for (size_t i = 0; i < inputs.size(); i++) { + uint64_t s = inputs[i]->file_size; + total += s; + if (total >= limit) { + inputs.resize(i + 1); + break; + } } } From bf60efcef29d2c9da0a6a01f40cc16cb92af0dce Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 15 Jun 2013 01:28:18 -0700 Subject: [PATCH 4/8] We have to dispatch RPCServer operations onto a strand, otherwise an error while both a read and write are pending can crash. Conflicts: src/cpp/ripple/RPCServer.cpp src/cpp/ripple/RPCServer.h --- src/cpp/ripple/RPCServer.cpp | 128 ++++++++++++++++++----------------- src/cpp/ripple/RPCServer.h | 1 + 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/src/cpp/ripple/RPCServer.cpp b/src/cpp/ripple/RPCServer.cpp index 8336df025..1486d26d9 100644 --- a/src/cpp/ripple/RPCServer.cpp +++ b/src/cpp/ripple/RPCServer.cpp @@ -19,16 +19,16 @@ SETUP_LOG (RPCServer) RPCServer::RPCServer(boost::asio::io_service& io_service, boost::asio::ssl::context& context, NetworkOPs* nopNetwork) - : mNetOps(nopNetwork), mSocket(io_service, context) + : mNetOps(nopNetwork), mSocket(io_service, context), mStrand(io_service) { mRole = RPCHandler::GUEST; } void RPCServer::connected() { - //std::cerr << "RPC request" << std::endl; - boost::asio::async_read_until(mSocket, mLineBuffer, "\r\n", - boost::bind(&RPCServer::handle_read_line, shared_from_this(), boost::asio::placeholders::error)); + //std::cerr << "RPC request" << std::endl; + boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", + mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), boost::asio::placeholders::error))); } void RPCServer::handle_read_req(const boost::system::error_code& e) @@ -48,8 +48,8 @@ void RPCServer::handle_read_req(const boost::system::error_code& e) else mReplyStr = handleRequest(req); - boost::asio::async_write(mSocket, boost::asio::buffer(mReplyStr), - boost::bind(&RPCServer::handle_write, shared_from_this(), boost::asio::placeholders::error)); + boost::asio::async_write (mSocket, boost::asio::buffer (mReplyStr), + mStrand.wrap (boost::bind (&RPCServer::handle_write, shared_from_this (), boost::asio::placeholders::error))); } static void dummy_handler() @@ -59,50 +59,53 @@ static void dummy_handler() void RPCServer::handle_read_line(const boost::system::error_code& e) { - if (e) - return; + if (e) + return; - HTTPRequestAction action = mHTTPRequest.consume(mLineBuffer); + HTTPRequestAction action = mHTTPRequest.consume (mLineBuffer); - if (action == haDO_REQUEST) - { // request with no body - WriteLog (lsWARNING, RPCServer) << "RPC HTTP request with no body"; - mSocket.async_shutdown(boost::bind(&dummy_handler)); - return; - } - else if (action == haREAD_LINE) - { - boost::asio::async_read_until(mSocket, mLineBuffer, "\r\n", - boost::bind(&RPCServer::handle_read_line, shared_from_this(), - boost::asio::placeholders::error)); - } - else if (action == haREAD_RAW) - { - int rLen = mHTTPRequest.getDataSize(); - if ((rLen < 0) || (rLen > RPC_MAXIMUM_QUERY)) - { - WriteLog (lsWARNING, RPCServer) << "Illegal RPC request length " << rLen; - mSocket.async_shutdown(boost::bind(&dummy_handler)); - return; - } + if (action == haDO_REQUEST) + { + // request with no body + WriteLog (lsWARNING, RPCServer) << "RPC HTTP request with no body"; + mSocket.async_shutdown (boost::bind (&dummy_handler)); + return; + } + else if (action == haREAD_LINE) + { + boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", + mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), + boost::asio::placeholders::error))); + } + else if (action == haREAD_RAW) + { + int rLen = mHTTPRequest.getDataSize (); - int alreadyHave = mLineBuffer.size(); + if ((rLen < 0) || (rLen > RPC_MAXIMUM_QUERY)) + { + WriteLog (lsWARNING, RPCServer) << "Illegal RPC request length " << rLen; + mSocket.async_shutdown (boost::bind (&dummy_handler)); + return; + } - if (alreadyHave < rLen) - { - mQueryVec.resize(rLen - alreadyHave); - boost::asio::async_read(mSocket, boost::asio::buffer(mQueryVec), - boost::bind(&RPCServer::handle_read_req, shared_from_this(), boost::asio::placeholders::error)); - WriteLog (lsTRACE, RPCServer) << "Waiting for completed request: " << rLen; - } - else - { // we have the whole thing - mQueryVec.resize(0); - handle_read_req(e); - } - } - else - mSocket.async_shutdown(boost::bind(&dummy_handler)); + int alreadyHave = mLineBuffer.size (); + + if (alreadyHave < rLen) + { + mQueryVec.resize (rLen - alreadyHave); + boost::asio::async_read (mSocket, boost::asio::buffer (mQueryVec), + mStrand.wrap (boost::bind (&RPCServer::handle_read_req, shared_from_this (), boost::asio::placeholders::error))); + WriteLog (lsTRACE, RPCServer) << "Waiting for completed request: " << rLen; + } + else + { + // we have the whole thing + mQueryVec.resize (0); + handle_read_req (e); + } + } + else + mSocket.async_shutdown (boost::bind (&dummy_handler)); } std::string RPCServer::handleRequest(const std::string& requestStr) @@ -181,24 +184,25 @@ bool RPCServer::parseAcceptRate(const std::string& sAcceptRate) void RPCServer::handle_write(const boost::system::error_code& e) { - //std::cerr << "async_write complete " << e << std::endl; + //std::cerr << "async_write complete " << e << std::endl; - if (!e) - { - HTTPRequestAction action = mHTTPRequest.requestDone(false); - if (action == haCLOSE_CONN) - mSocket.async_shutdown(boost::bind(&dummy_handler)); - else - { - boost::asio::async_read_until(mSocket, mLineBuffer, "\r\n", - boost::bind(&RPCServer::handle_read_line, shared_from_this(), boost::asio::placeholders::error)); - } - } + if (!e) + { + HTTPRequestAction action = mHTTPRequest.requestDone (false); - if (e != boost::asio::error::operation_aborted) - { - //connection_manager_.stop(shared_from_this()); - } + if (action == haCLOSE_CONN) + mSocket.async_shutdown (boost::bind (&dummy_handler)); + else + { + boost::asio::async_read_until (mSocket, mLineBuffer, "\r\n", + mStrand.wrap (boost::bind (&RPCServer::handle_read_line, shared_from_this (), boost::asio::placeholders::error))); + } + } + + if (e != boost::asio::error::operation_aborted) + { + //connection_manager_.stop(shared_from_this()); + } } // vim:ts=4 diff --git a/src/cpp/ripple/RPCServer.h b/src/cpp/ripple/RPCServer.h index 785514418..ac337e89e 100644 --- a/src/cpp/ripple/RPCServer.h +++ b/src/cpp/ripple/RPCServer.h @@ -16,6 +16,7 @@ private: NetworkOPs* mNetOps; AutoSocket mSocket; + boost::asio::io_service::strand mStrand; boost::asio::streambuf mLineBuffer; Blob mQueryVec; From 91a6f964afa76cd0a7c5dd59defadc6d26c6794c Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 15 Jun 2013 01:41:18 -0700 Subject: [PATCH 5/8] In AutoSocket::async_shutdown, pass the error code to the handler, don't throw it. --- src/cpp/ripple/AutoSocket.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/AutoSocket.h b/src/cpp/ripple/AutoSocket.h index e9dbf5b6b..8e5cfdce6 100644 --- a/src/cpp/ripple/AutoSocket.h +++ b/src/cpp/ripple/AutoSocket.h @@ -107,8 +107,16 @@ public: mSocket->async_shutdown(handler); else { - lowest_layer().shutdown(plain_socket::shutdown_both); - mSocket->get_io_service().post(boost::bind(handler, error_code())); + error_code ec; + try + { + lowest_layer().shutdown(plain_socket::shutdown_both); + } + catch (boost::system::system_error& e) + { + ec = e.code(); + } + mSocket->get_io_service().post(boost::bind(handler, ec)); } } From 91e1b17e6130d94b7cd70d6c69cb279a9a7b8ce3 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 15 Jun 2013 02:08:09 -0700 Subject: [PATCH 6/8] Allow larger mmaps. --- src/cpp/ripple/DBInit.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/DBInit.cpp b/src/cpp/ripple/DBInit.cpp index d8daa6a32..bd63786bf 100644 --- a/src/cpp/ripple/DBInit.cpp +++ b/src/cpp/ripple/DBInit.cpp @@ -8,7 +8,7 @@ const char *TxnDBInit[] = { "PRAGMA journal_size_limit=1582080;", #if (ULONG_MAX > UINT_MAX) && !defined (NO_SQLITE_MMAP) - "PRAGMA mmap_size=4294967296;", + "PRAGMA mmap_size=17179869184;", #endif "BEGIN TRANSACTION;", @@ -279,7 +279,7 @@ const char *HashNodeDBInit[] = { "PRAGMA journal_size_limit=1582080;", #if (ULONG_MAX > UINT_MAX) && !defined (NO_SQLITE_MMAP) - "PRAGMA mmap_size=4294967296;", + "PRAGMA mmap_size=171798691840;", #endif "BEGIN TRANSACTION;", From 4473b85eb7aa96c13c4c726ea705076904237597 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 15 Jun 2013 04:07:42 -0700 Subject: [PATCH 7/8] Fix crash bug on missing root. --- src/cpp/ripple/ripple_SHAMap.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/ripple_SHAMap.cpp b/src/cpp/ripple/ripple_SHAMap.cpp index 171566813..58badc3a2 100644 --- a/src/cpp/ripple/ripple_SHAMap.cpp +++ b/src/cpp/ripple/ripple_SHAMap.cpp @@ -769,8 +769,10 @@ bool SHAMap::fetchRoot(uint256 const& hash, SHAMapSyncFilter* filter) else WriteLog (lsTRACE, SHAMap) << "Fetch root SHAMap node " << hash; } - root = fetchNodeExternalNT(SHAMapNode(), hash); - if (!root) + Ledger::pointer newRoot = fetchNodeExternalNT(SHAMapNode(), hash); + if (newRoot) + root = newRoot; + else { Blob nodeData; if (!filter || !filter->haveNode(SHAMapNode(), hash, nodeData)) From baf9279909ba18ca6269ae0143b4ebf2779e3c34 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Sat, 15 Jun 2013 04:13:57 -0700 Subject: [PATCH 8/8] Typo. --- src/cpp/ripple/ripple_SHAMap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/ripple_SHAMap.cpp b/src/cpp/ripple/ripple_SHAMap.cpp index 58badc3a2..beb1508f5 100644 --- a/src/cpp/ripple/ripple_SHAMap.cpp +++ b/src/cpp/ripple/ripple_SHAMap.cpp @@ -769,7 +769,7 @@ bool SHAMap::fetchRoot(uint256 const& hash, SHAMapSyncFilter* filter) else WriteLog (lsTRACE, SHAMap) << "Fetch root SHAMap node " << hash; } - Ledger::pointer newRoot = fetchNodeExternalNT(SHAMapNode(), hash); + SHAMapTreeNode::pointer newRoot = fetchNodeExternalNT(SHAMapNode(), hash); if (newRoot) root = newRoot; else