From 9c0310280b276ff56de47c949370fc12fcbc7f11 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Tue, 19 Mar 2013 21:15:08 -0700 Subject: [PATCH 01/42] RPC account_info support specifying ledger. --- src/cpp/ripple/CallRPC.cpp | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/src/cpp/ripple/CallRPC.cpp b/src/cpp/ripple/CallRPC.cpp index 1d0f009105..ef23c387a9 100644 --- a/src/cpp/ripple/CallRPC.cpp +++ b/src/cpp/ripple/CallRPC.cpp @@ -65,19 +65,11 @@ std::string EncodeBase64(const std::string& s) // TODO New routine for parsing ledger parameters, other routines should standardize on this. static bool jvParseLedger(Json::Value& jvRequest, const std::string& strLedger) { - if (strLedger == "closed") + if (strLedger == "current" || strLedger == "closed" || strLedger == "validated") { - jvRequest["ledger_index"] = -1; + jvRequest["ledger_index"] = strLedger; } - else if (strLedger == "current") - { - jvRequest["ledger_index"] = -2; - } - else if (strLedger == "validated") - { - jvRequest["ledger_index"] = -3; - } - else if (strLedger.length() > 12) + else if (strLedger.length() == 32) { // YYY Could confirm this is a uint256. jvRequest["ledger_hash"] = strLedger; @@ -143,7 +135,7 @@ Json::Value RPCParser::parseInternal(const Json::Value& jvParams) } // account_info || -// account_info || [] +// account_info || [[] ] Json::Value RPCParser::parseAccountInfo(const Json::Value& jvParams) { Json::Value jvRequest(Json::objectValue); @@ -159,6 +151,9 @@ Json::Value RPCParser::parseAccountInfo(const Json::Value& jvParams) jvRequest["ident"] = strIdent; jvRequest["account_index"] = iIndex; + if (jvParams.size() == 3 && !jvParseLedger(jvRequest, jvParams[2u].asString())) + return rpcError(rpcLGR_IDX_MALFORMED); + return jvRequest; } @@ -343,20 +338,7 @@ Json::Value RPCParser::parseLedger(const Json::Value& jvParams) return jvRequest; } - std::string strLedger = jvParams[0u].asString(); - - if (strLedger == "current" || strLedger == "closed" || strLedger == "validated") - { - jvRequest["ledger_index"] = strLedger; - } - else if (strLedger.length() > 12) - { - jvRequest["ledger_hash"] = strLedger; - } - else - { - jvRequest["ledger_index"] = lexical_cast_s(strLedger); - } + jvParseLedger(jvRequest, jvParams[0u].asString()); if (2 == jvParams.size() && jvParams[1u].asString() == "full") { @@ -373,7 +355,7 @@ Json::Value RPCParser::parseLedgerId(const Json::Value& jvParams) std::string strLedger = jvParams[0u].asString(); - if (strLedger.length() > 32) + if (strLedger.length() == 32) { jvRequest["ledger_hash"] = strLedger; } @@ -641,7 +623,7 @@ Json::Value RPCParser::parseCommand(std::string strMethod, Json::Value jvParams) // Request-response methods // - Returns an error, or the request. // - To modify the method, provide a new method in the request. - { "account_info", &RPCParser::parseAccountInfo, 1, 2 }, + { "account_info", &RPCParser::parseAccountInfo, 1, 3 }, { "account_lines", &RPCParser::parseAccountItems, 1, 2 }, { "account_offers", &RPCParser::parseAccountItems, 1, 2 }, { "account_tx", &RPCParser::parseAccountTransactions, 2, 4 }, From d4f30bab44043dc2129e8dfa1dd5cdb27e07de57 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Tue, 19 Mar 2013 21:15:45 -0700 Subject: [PATCH 02/42] More enhancements for browser.js --- bin/browser.js | 119 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 101 insertions(+), 18 deletions(-) diff --git a/bin/browser.js b/bin/browser.js index 30e83c3c36..d02aeb2b3b 100755 --- a/bin/browser.js +++ b/bin/browser.js @@ -18,6 +18,7 @@ // L=current | closed | validated | index | hash // +var async = require("async"); var extend = require("extend"); var http = require("http"); var url = require("url"); @@ -224,10 +225,9 @@ var rewrite_object = function (obj, opts) { // Its a ledger entry. if (obj.node.LedgerEntryType === 'AccountRoot') { - out.node.Account = rewrite_type('address', obj.node.Account, opts); -// out.node.hash = rewrite_type('transaction', obj.node.hash, opts); - out.node.PreviousTxnID = rewrite_type('transaction', out.node.PreviousTxnID, opts); - out.node.PreviousTxnLgrSeq = rewrite_type('ledger', out.node.PreviousTxnLgrSeq, opts); + rewrite_field('address', out.node, 'Account', opts); + rewrite_field('transaction', out.node, 'PreviousTxnID', opts); + rewrite_field('ledger', out.node, 'PreviousTxnLgrSeq', opts); } out.node.LedgerEntryType = '' + out.node.LedgerEntryType + ''; @@ -236,6 +236,86 @@ var rewrite_object = function (obj, opts) { return out; }; +var augment_object = function (obj, opts, done) { + if (obj.node.LedgerEntryType == 'AccountRoot') { + var tx_hash = obj.node.PreviousTxnID; + var tx_ledger = obj.node.PreviousTxnLgrSeq; + + obj.history = []; + + async.whilst( + function () { return tx_hash; }, + function (callback) { +// console.log("augment_object: request: %s %s", tx_hash, tx_ledger); + opts.remote.request_tx(tx_hash) + .on('success', function (m) { + tx_hash = undefined; + tx_ledger = undefined; + +//console.log("augment_object: ", JSON.stringify(m)); + m.meta.AffectedNodes.filter(function(n) { +// console.log("augment_object: ", JSON.stringify(n)); +// if (n.ModifiedNode) +// console.log("augment_object: %s %s %s %s %s %s/%s", 'ModifiedNode' in n, n.ModifiedNode && (n.ModifiedNode.LedgerEntryType === 'AccountRoot'), n.ModifiedNode && n.ModifiedNode.FinalFields && (n.ModifiedNode.FinalFields.Account === obj.node.Account), Object.keys(n)[0], n.ModifiedNode && (n.ModifiedNode.LedgerEntryType), obj.node.Account, n.ModifiedNode && n.ModifiedNode.FinalFields && n.ModifiedNode.FinalFields.Account); +// if ('ModifiedNode' in n && n.ModifiedNode.LedgerEntryType === 'AccountRoot') +// { +// console.log("***: ", JSON.stringify(m)); +// console.log("***: ", JSON.stringify(n)); +// } + return 'ModifiedNode' in n + && n.ModifiedNode.LedgerEntryType === 'AccountRoot' + && n.ModifiedNode.FinalFields + && n.ModifiedNode.FinalFields.Account === obj.node.Account; + }) + .forEach(function (n) { + tx_hash = n.ModifiedNode.PreviousTxnID; + tx_ledger = n.ModifiedNode.PreviousTxnLgrSeq; + + obj.history.push({ + tx_hash: tx_hash, + tx_ledger: tx_ledger + }); +console.log("augment_object: next: %s %s", tx_hash, tx_ledger); + }); + + callback(); + }) + .on('error', function (m) { + callback(m); + }) + .request(); + }, + function (err) { + if (err) { + done(); + } + else { + async.forEach(obj.history, function (o, callback) { + opts.remote.request_account_info(obj.node.Account) + .ledger_index(o.tx_ledger) + .on('success', function (m) { +//console.log("augment_object: ", JSON.stringify(m)); + o.Balance = m.account_data.Balance; +// o.account_data = m.account_data; + callback(); + }) + .on('error', function (m) { + o.error = m; + callback(); + }) + .request(); + }, + function (err) { + done(err); + }); + } + }); + } + else { + done(); + } +}; + if (process.argv.length < 4 || process.argv.length > 7) { console.log("Usage: %s ws_ip ws_port [ [ []]]", program); } @@ -248,15 +328,10 @@ else { // console.log("START"); var self = this; - self.base = { - hostname: ip, - port: port, - }; - var remote = (new Remote({ websocket_ip: ws_ip, websocket_port: ws_port, -// trace: true + trace: false })) .on('state', function (m) { console.log("STATE: %s", m); @@ -267,6 +342,12 @@ else { .connect() ; + self.base = { + hostname: ip, + port: port, + remote: remote, + }; + // console.log("SERVE"); var server = http.createServer(function (req, res) { var input = ""; @@ -294,14 +375,16 @@ else { .on('success', function (m) { // console.log("account_root: %s", JSON.stringify(m, undefined, 2)); - httpd_response(res, - { - statusCode: 200, - url: _url, - body: "
"
-                            + JSON.stringify(rewrite_object(m, self.base), undefined, 2)
-                            + "
" - }); + augment_object(m, self.base, function() { + httpd_response(res, + { + statusCode: 200, + url: _url, + body: "
"
+                              + JSON.stringify(rewrite_object(m, self.base), undefined, 2)
+                              + "
" + }); + }); }) .request(); From 5494bc3158e29ac386d6b2a74c85f3fc7a7cfd9d Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Tue, 19 Mar 2013 23:04:47 -0700 Subject: [PATCH 03/42] Enlarge the SQLite database cache intelligently. --- src/cpp/database/SqliteDatabase.cpp | 9 ++++++++- src/cpp/database/SqliteDatabase.h | 3 ++- src/cpp/database/database.h | 3 ++- src/cpp/ripple/Application.cpp | 6 ++++++ src/cpp/ripple/Application.h | 6 ++++-- src/cpp/ripple/Config.cpp | 5 +++-- src/cpp/ripple/Config.h | 3 ++- src/cpp/ripple/RPCHandler.cpp | 14 ++++++++++++-- 8 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/cpp/database/SqliteDatabase.cpp b/src/cpp/database/SqliteDatabase.cpp index aa3195daf8..4bb0850bf7 100644 --- a/src/cpp/database/SqliteDatabase.cpp +++ b/src/cpp/database/SqliteDatabase.cpp @@ -196,11 +196,18 @@ uint64 SqliteDatabase::getBigInt(int colIndex) return(sqlite3_column_int64(mCurrentStmt, colIndex)); } -int SqliteDatabase::getKBUsed() +int SqliteDatabase::getKBUsedAll() { return static_cast(sqlite3_memory_used() / 1024); } +int SqliteDatabase::getKBUsedDB() +{ + int cur = 0, hiw = 0; + sqlite3_db_status(mConnection, SQLITE_DBSTATUS_CACHE_USED, &cur, &hiw, 0); + return cur / 1024; +} + static int SqliteWALHook(void *s, sqlite3* dbCon, const char *dbName, int walSize) { (reinterpret_cast(s))->doHook(dbName, walSize); diff --git a/src/cpp/database/SqliteDatabase.h b/src/cpp/database/SqliteDatabase.h index 2cd9896139..ed2d829457 100644 --- a/src/cpp/database/SqliteDatabase.h +++ b/src/cpp/database/SqliteDatabase.h @@ -58,7 +58,8 @@ public: void runWal(); void doHook(const char *db, int walSize); - int getKBUsed(); + int getKBUsedDB(); + int getKBUsedAll(); }; class SqliteStatement diff --git a/src/cpp/database/database.h b/src/cpp/database/database.h index 579ed2929d..dca77106e5 100644 --- a/src/cpp/database/database.h +++ b/src/cpp/database/database.h @@ -89,7 +89,8 @@ public: virtual bool setupCheckpointing(JobQueue*) { return false; } virtual SqliteDatabase* getSqliteDB() { return NULL; } - virtual int getKBUsed() { return -1; } + virtual int getKBUsedAll() { return -1; } + virtual int getKBUsedDB() { return -1; } }; #endif diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index a9ff4ce797..e60851520c 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -24,8 +24,11 @@ LogPartition TaggedCachePartition("TaggedCache"); LogPartition AutoSocketPartition("AutoSocket"); Application* theApp = NULL; +int DatabaseCon::sCount = 0; + DatabaseCon::DatabaseCon(const std::string& strName, const char *initStrings[], int initCount) { + ++sCount; boost::filesystem::path pPath = (theConfig.RUN_STANDALONE && (theConfig.START_UP != Config::LOAD)) ? "" // Use temporary files. : (theConfig.DATA_DIR / strName); // Use regular db files. @@ -172,6 +175,9 @@ void Application::setup() mLedgerMaster.tune(theConfig.getSize(siLedgerSize), theConfig.getSize(siLedgerAge)); mLedgerMaster.setMinValidations(theConfig.VALIDATION_QUORUM); + theApp->getHashNodeDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % + (theConfig.getSize(siDBCache) * 1024))); + // // Allow peer connections. // diff --git a/src/cpp/ripple/Application.h b/src/cpp/ripple/Application.h index 48e58d97af..a467202665 100644 --- a/src/cpp/ripple/Application.h +++ b/src/cpp/ripple/Application.h @@ -36,12 +36,14 @@ class DatabaseCon protected: Database* mDatabase; boost::recursive_mutex mLock; + static int sCount; public: DatabaseCon(const std::string& name, const char *initString[], int countInit); ~DatabaseCon(); - Database* getDB() { return mDatabase; } - boost::recursive_mutex& getDBLock() { return mLock; } + Database* getDB() { return mDatabase; } + boost::recursive_mutex& getDBLock() { return mLock; } + static int getCount() { return sCount; } }; class Application diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index 8d7abeb317..f5cbd6d8e5 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -486,7 +486,7 @@ void Config::load() int Config::getSize(SizedItemName item) { - SizedItem sizeTable[] = { + SizedItem sizeTable[] = { // tiny small medium large huge { siSweepInterval, { 10, 30, 60, 90, 90 } }, { siLedgerFetch, { 2, 2, 3, 4, 5 } }, { siValidationsSize, { 256, 256, 512, 1024, 1024 } }, @@ -496,7 +496,8 @@ int Config::getSize(SizedItemName item) { siLedgerSize, { 32, 64, 128, 1024, 0 } }, { siLedgerAge, { 30, 60, 120, 300, 600 } }, { siLineCacheSize, { 8192, 32768, 131072, 1048576, 0 } }, - { siLineCacheAge, { 500, 600, 1800, 3600, 7200 } } + { siLineCacheAge, { 500, 600, 1800, 3600, 7200 } }, + { siDBCache, { 8, 12, 24, 64, 96 } }, }; for (int i = 0; i < (sizeof(sizeTable) / sizeof(SizedItem)); ++i) diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index 7d1a420c8b..6dd0fe3893 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -64,7 +64,8 @@ enum SizedItemName siLedgerAge, siLedgerFetch, siLineCacheSize, - siLineCacheAge + siLineCacheAge, + siDBCache }; struct SizedItem diff --git a/src/cpp/ripple/RPCHandler.cpp b/src/cpp/ripple/RPCHandler.cpp index d642e05e40..cc0a84f7ee 100644 --- a/src/cpp/ripple/RPCHandler.cpp +++ b/src/cpp/ripple/RPCHandler.cpp @@ -2021,9 +2021,19 @@ Json::Value RPCHandler::doGetCounts(Json::Value jvRequest, int& cost) BOOST_FOREACH(InstanceType::InstanceCount& it, count) ret[it.first] = it.second; - int dbKB = theApp->getLedgerDB()->getDB()->getKBUsed(); + int dbKB = theApp->getLedgerDB()->getDB()->getKBUsedAll(); if (dbKB > 0) - ret["dbKB"] = dbKB; + ret["dbKBTotal"] = dbKB; + + dbKB = theApp->getLedgerDB()->getDB()->getKBUsedDB(); + if (dbKB > 0) + ret["dbKBLedger"] = dbKB; + dbKB = theApp->getHashNodeDB()->getDB()->getKBUsedDB(); + if (dbKB > 0) + ret["dbKBHashNode"] = dbKB; + dbKB = theApp->getTxnDB()->getDB()->getKBUsedDB(); + if (dbKB > 0) + ret["dbKBTransaction"] = dbKB; std::string uptime; int s = upTime(); From ce1e1a5bcaaad676390342df6fdb97895f4ec796 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 00:02:52 -0700 Subject: [PATCH 04/42] Don't let fetches squeeze memory on tiny and small builds. --- src/cpp/ripple/Ledger.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index d9fdab4ca5..fd34ddd4a0 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -500,10 +500,8 @@ void Ledger::saveAcceptedLedger(Job&, bool fromConsensus) mCloseResolution % mCloseFlags % mAccountHash.GetHex() % mTransHash.GetHex())); } -#if 0 - if (!fromConsensus) + if (!fromConsensus && (theConfig.NODE_SIZE < 2)) // tiny or small dropCache(); -#endif if (theApp->getJobQueue().getJobCountTotal(jtPUBOLDLEDGER) < 2) theApp->getLedgerMaster().resumeAcquiring(); From 9fbbee318df82346608eb9f5ed3dcbe9e2a528e2 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 01:13:07 -0700 Subject: [PATCH 05/42] Tweak the sqlite cache sizing. --- src/cpp/ripple/Application.cpp | 2 +- src/cpp/ripple/Config.cpp | 2 +- src/cpp/ripple/Config.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index e60851520c..2302c6a631 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -176,7 +176,7 @@ void Application::setup() mLedgerMaster.setMinValidations(theConfig.VALIDATION_QUORUM); theApp->getHashNodeDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % - (theConfig.getSize(siDBCache) * 1024))); + (theConfig.getSize(siHashNodeDBCache) * 1024))); // // Allow peer connections. diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index f5cbd6d8e5..8afef28738 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -497,7 +497,7 @@ int Config::getSize(SizedItemName item) { siLedgerAge, { 30, 60, 120, 300, 600 } }, { siLineCacheSize, { 8192, 32768, 131072, 1048576, 0 } }, { siLineCacheAge, { 500, 600, 1800, 3600, 7200 } }, - { siDBCache, { 8, 12, 24, 64, 96 } }, + { siHashNodeDBCache, { 24, 48, 64, 128, 256 } }, }; for (int i = 0; i < (sizeof(sizeTable) / sizeof(SizedItem)); ++i) diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index 6dd0fe3893..0ceceba598 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -65,7 +65,7 @@ enum SizedItemName siLedgerFetch, siLineCacheSize, siLineCacheAge, - siDBCache + siHashNodeDBCache }; struct SizedItem From 83e347c839905a45e11684c10706f06770c0679d Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 20 Mar 2013 01:21:41 -0700 Subject: [PATCH 06/42] #27 Error creating offer without auth. --- src/cpp/ripple/OfferCreateTransactor.cpp | 11 +++++++++++ src/cpp/ripple/TransactionErr.cpp | 1 + src/cpp/ripple/TransactionErr.h | 1 + test/path-test.js | 4 ++-- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/OfferCreateTransactor.cpp b/src/cpp/ripple/OfferCreateTransactor.cpp index afff906994..686b21e832 100644 --- a/src/cpp/ripple/OfferCreateTransactor.cpp +++ b/src/cpp/ripple/OfferCreateTransactor.cpp @@ -426,6 +426,17 @@ TER OfferCreateTransactor::doApply() terResult = terNO_ACCOUNT; } + else if (isSetBit(sleTakerPays->getFieldU32(sfFlags), lsfRequireAuth)) { + SLE::pointer sleRippleState = mEngine->entryCache(ltRIPPLE_STATE, Ledger::getRippleStateIndex(mTxnAccountID, uPaysIssuerID, uPaysCurrency)); + bool bHigh = mTxnAccountID > uPaysIssuerID; + + if (!sleRippleState + || !isSetBit(sleRippleState->getFieldU32(sfFlags), (bHigh ? lsfHighAuth : lsfLowAuth))) { + cLog(lsWARNING) << "OfferCreate: delay: can't receive IOUs from issuer without auth."; + + terResult = terNO_AUTH; + } + } } STAmount saPaid; diff --git a/src/cpp/ripple/TransactionErr.cpp b/src/cpp/ripple/TransactionErr.cpp index a605a2f49a..160064ffeb 100644 --- a/src/cpp/ripple/TransactionErr.cpp +++ b/src/cpp/ripple/TransactionErr.cpp @@ -82,6 +82,7 @@ bool transResultInfo(TER terCode, std::string& strToken, std::string& strHuman) { terFUNDS_SPENT, "terFUNDS_SPENT", "Can't set password, password set funds already spent." }, { terINSUF_FEE_B, "terINSUF_FEE_B", "Account balance can't pay fee." }, { terNO_ACCOUNT, "terNO_ACCOUNT", "The source account does not exist." }, + { terNO_AUTH, "terNO_AUTH", "Not authorized to hold IOUs." }, { terNO_LINE, "terNO_LINE", "No such line." }, { terPRE_SEQ, "terPRE_SEQ", "Missing/inapplicable prior transaction." }, { terOWNERS, "terOWNERS", "Non-zero owner count." }, diff --git a/src/cpp/ripple/TransactionErr.h b/src/cpp/ripple/TransactionErr.h index a1d9fb6909..fe54d9f4bb 100644 --- a/src/cpp/ripple/TransactionErr.h +++ b/src/cpp/ripple/TransactionErr.h @@ -95,6 +95,7 @@ enum TER // aka TransactionEngineResult terFUNDS_SPENT, // This is a free transaction, therefore don't burden network. terINSUF_FEE_B, // Can't pay fee, therefore don't burden network. terNO_ACCOUNT, // Can't pay fee, therefore don't burden network. + terNO_AUTH, // Not authorized to hold IOUs. terNO_LINE, // Internal flag. terOWNERS, // Can't succeed with non-zero owner count. terPRE_SEQ, // Can't pay fee, no point in forwarding, therefore don't burden network. diff --git a/test/path-test.js b/test/path-test.js index 59ffd8cc0d..ee75afbd67 100644 --- a/test/path-test.js +++ b/test/path-test.js @@ -1189,8 +1189,8 @@ buster.testCase("Indirect paths", { }); buster.testCase("Quality paths", { - // 'setUp' : testutils.build_setup(), - 'setUp' : testutils.build_setup({ verbose: true }), + 'setUp' : testutils.build_setup(), + // 'setUp' : testutils.build_setup({ verbose: true }), // 'setUp' : testutils.build_setup({ verbose: true, no_server: true }), 'tearDown' : testutils.build_teardown(), From bd5c67328fcf77851fcb2bf4d064999d9f167ffe Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 01:34:52 -0700 Subject: [PATCH 07/42] Ledger hashes are 64 characters long. --- src/cpp/ripple/CallRPC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/CallRPC.cpp b/src/cpp/ripple/CallRPC.cpp index ef23c387a9..03ed58a536 100644 --- a/src/cpp/ripple/CallRPC.cpp +++ b/src/cpp/ripple/CallRPC.cpp @@ -69,7 +69,7 @@ static bool jvParseLedger(Json::Value& jvRequest, const std::string& strLedger) { jvRequest["ledger_index"] = strLedger; } - else if (strLedger.length() == 32) + else if (strLedger.length() == 64) { // YYY Could confirm this is a uint256. jvRequest["ledger_hash"] = strLedger; From af8418c89ccf8b0b1923b22a6b55e58b30b371d3 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 20 Mar 2013 01:50:22 -0700 Subject: [PATCH 08/42] #27 Error rippling without auth. --- src/cpp/ripple/RippleCalc.cpp | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/cpp/ripple/RippleCalc.cpp b/src/cpp/ripple/RippleCalc.cpp index 213923acf2..b8a0b0f0ed 100644 --- a/src/cpp/ripple/RippleCalc.cpp +++ b/src/cpp/ripple/RippleCalc.cpp @@ -114,7 +114,7 @@ TER PathState::pushImply( // Append a node and insert before it any implied nodes. // Offers may go back to back. -// <-- terResult: tesSUCCESS, temBAD_PATH, terNO_LINE, tecPATH_DRY +// <-- terResult: tesSUCCESS, temBAD_PATH, terNO_ACCOUNT, terNO_AUTH, terNO_LINE, tecPATH_DRY TER PathState::pushNode( const int iType, const uint160& uAccountID, @@ -231,11 +231,30 @@ TER PathState::pushNode( << STAmount::createHumanCurrency(pnCur.uCurrencyID) << "." ; - STAmount saOwed = lesEntries.rippleOwed(pnCur.uAccountID, pnBck.uAccountID, pnCur.uCurrencyID); + SLE::pointer sleBck = lesEntries.entryCache(ltACCOUNT_ROOT, Ledger::getAccountRootIndex(pnBck.uAccountID)); + bool bHigh = pnBck.uAccountID > pnCur.uAccountID; - if (!saOwed.isPositive() && -saOwed >= lesEntries.rippleLimit(pnCur.uAccountID, pnBck.uAccountID, pnCur.uCurrencyID)) + if (!sleBck) { - terResult = tecPATH_DRY; + cLog(lsWARNING) << "pushNode: delay: can't receive IOUs from non-existent issuer: " << RippleAddress::createHumanAccountID(pnBck.uAccountID); + + terResult = terNO_ACCOUNT; + } + else if (isSetBit(sleBck->getFieldU32(sfFlags), lsfRequireAuth) + && !isSetBit(sleRippleState->getFieldU32(sfFlags), (bHigh ? lsfHighAuth : lsfLowAuth))) { + cLog(lsWARNING) << "pushNode: delay: can't receive IOUs from issuer without auth."; + + terResult = terNO_AUTH; + } + + if (tesSUCCESS == terResult) + { + STAmount saOwed = lesEntries.rippleOwed(pnCur.uAccountID, pnBck.uAccountID, pnCur.uCurrencyID); + + if (!saOwed.isPositive() && -saOwed >= lesEntries.rippleLimit(pnCur.uAccountID, pnBck.uAccountID, pnCur.uCurrencyID)) + { + terResult = tecPATH_DRY; + } } } } @@ -287,7 +306,7 @@ TER PathState::pushNode( // Set to an expanded path. // -// terStatus = tesSUCCESS, temBAD_PATH, terNO_LINE, or temBAD_PATH_LOOP +// terStatus = tesSUCCESS, temBAD_PATH, terNO_LINE, terNO_ACCOUNT, terNO_AUTH, or temBAD_PATH_LOOP void PathState::setExpanded( const LedgerEntrySet& lesSource, const STPath& spSourcePath, From 8dc9b7bc64a2b6c71d9daf030db68904fd307fbb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 02:00:40 -0700 Subject: [PATCH 09/42] Log IP address on peer disconnection. --- src/cpp/ripple/Peer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index e4a418f0f7..ee0077beb7 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -384,7 +384,7 @@ void Peer::handleReadHeader(const boost::system::error_code& error) } else { - cLog(lsINFO) << "Peer: Header: Error: " << ADDRESS(this) << ": " << error.category().name() << ": " << error.message() << ": " << error; + cLog(lsINFO) << "Peer: Header: Error: " << getIP() << ": " << error.category().name() << ": " << error.message() << ": " << error; detach("hrh2", true); } } From d8e3cb1217d8c3fe63100ecdd04bf2640fcbd47e Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 02:01:33 -0700 Subject: [PATCH 10/42] Log when we attempt a connection. --- src/cpp/ripple/ConnectionPool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/ConnectionPool.cpp b/src/cpp/ripple/ConnectionPool.cpp index d17f38885e..8faacaff16 100644 --- a/src/cpp/ripple/ConnectionPool.cpp +++ b/src/cpp/ripple/ConnectionPool.cpp @@ -319,7 +319,7 @@ Peer::pointer ConnectionPool::peerConnect(const std::string& strIp, int iPort) if (ppResult) { ppResult->connect(strIp, iPort); - cLog(lsTRACE) << "Pool: Connecting: " << ADDRESS_SHARED(ppResult) << ": " << strIp << " " << iPort; + cLog(lsDEBUG) << "Pool: Connecting: " << strIp << " " << iPort; } else { From 9ceb380cb83e2879fd953706252206ad5ddadace Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 05:41:08 -0700 Subject: [PATCH 11/42] Make debug easier by naming threads. --- src/cpp/ripple/JobQueue.cpp | 2 ++ src/cpp/ripple/LoadManager.cpp | 1 + src/cpp/ripple/WSDoor.cpp | 1 + src/cpp/ripple/main.cpp | 2 ++ src/cpp/ripple/utils.cpp | 44 ++++++++++++++++++++++++++++++++-- src/cpp/ripple/utils.h | 2 ++ 6 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/JobQueue.cpp b/src/cpp/ripple/JobQueue.cpp index 8e2d618a6e..303035fe44 100644 --- a/src/cpp/ripple/JobQueue.cpp +++ b/src/cpp/ripple/JobQueue.cpp @@ -266,6 +266,7 @@ void JobQueue::threadEntry() boost::mutex::scoped_lock sl(mJobLock); while (1) { + NameThread("waiting"); while (mJobSet.empty() && !mShuttingDown) mJobCond.wait(sl); @@ -286,6 +287,7 @@ void JobQueue::threadEntry() ++(mJobCounts[type].second); sl.unlock(); + NameThread(Job::toString(type)); cLog(lsTRACE) << "Doing " << Job::toString(type) << " job"; job.doJob(); } // must destroy job without holding lock diff --git a/src/cpp/ripple/LoadManager.cpp b/src/cpp/ripple/LoadManager.cpp index 41d15ac94c..56503fff5c 100644 --- a/src/cpp/ripple/LoadManager.cpp +++ b/src/cpp/ripple/LoadManager.cpp @@ -317,6 +317,7 @@ int LoadManager::getUptime() void LoadManager::threadEntry() { + NameThread("loadMgr"); boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time(); while (1) { diff --git a/src/cpp/ripple/WSDoor.cpp b/src/cpp/ripple/WSDoor.cpp index ec4a84e89d..40b1939f1a 100644 --- a/src/cpp/ripple/WSDoor.cpp +++ b/src/cpp/ripple/WSDoor.cpp @@ -48,6 +48,7 @@ static DH* handleTmpDh(SSL* ssl, int is_export, int iKeyLength) void WSDoor::startListening() { + NameThread("websocket"); // Generate a single SSL context for use by all connections. boost::shared_ptr mCtx; mCtx = boost::make_shared(boost::asio::ssl::context::sslv23); diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index c791d183fc..320149a036 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -127,6 +127,7 @@ void printHelp(const po::options_description& desc) int main(int argc, char* argv[]) { + NameThread("main"); int iResult = 0; po::variables_map vm; // Map of options. @@ -248,6 +249,7 @@ int main(int argc, char* argv[]) { // No arguments. Run server. setupServer(); + NameThread("io"); startServer(); } else diff --git a/src/cpp/ripple/utils.cpp b/src/cpp/ripple/utils.cpp index 4a67a2311e..db1754af8a 100644 --- a/src/cpp/ripple/utils.cpp +++ b/src/cpp/ripple/utils.cpp @@ -1,5 +1,9 @@ -#include "utils.h" -#include "uint256.h" + +#ifdef __linux__ +#include +#endif + +#include #include #include @@ -9,6 +13,9 @@ #include +#include "utils.h" +#include "uint256.h" + void getRand(unsigned char *buf, int num) { if (RAND_bytes(buf, num) != 1) @@ -333,6 +340,39 @@ uint32_t be32toh(uint32_t value) #endif +#ifdef PR_SET_NAME +#define HAVE_NAME_THREAD +extern void NameThread(const char* n) +{ + static std::string pName; + + if (pName.empty()) + { + std::ifstream cLine("/proc/self/cmdline", std::ios::in); + cLine >> pName; + if (pName.empty()) + pName = "rippled"; + else + { + size_t zero = pName.find_first_of('\0'); + if ((zero != std::string::npos) && (zero != 0)) + pName = pName.substr(0, zero - 1); + size_t slash = pName.find_last_of('/'); + if (slash != std::string::npos) + pName = pName.substr(slash + 1); + } + pName += " "; + } + + prctl(PR_SET_NAME, (pName + n).c_str(), 0, 0, 0); +} +#endif + +#ifndef HAVE_NAME_THREAD +extern void NameThread(const char*) +{ ; } +#endif + BOOST_AUTO_TEST_SUITE( Utils) BOOST_AUTO_TEST_CASE( ParseUrl ) diff --git a/src/cpp/ripple/utils.h b/src/cpp/ripple/utils.h index 6dcea25209..8cc6014308 100644 --- a/src/cpp/ripple/utils.h +++ b/src/cpp/ripple/utils.h @@ -286,6 +286,8 @@ template T range_check_cast(const U& value, const T& min bool parseUrl(const std::string& strUrl, std::string& strScheme, std::string& strDomain, int& iPort, std::string& strPath); +extern void NameThread(const char *); + #if (!defined(FORCE_NO_C11X) && (__cplusplus > 201100L)) || defined(FORCE_C11X) #define C11X From 9be2e626911485b623f39817edb09e2fd4fa3f87 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 05:42:46 -0700 Subject: [PATCH 12/42] Off by one in previous patch. --- src/cpp/ripple/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/utils.cpp b/src/cpp/ripple/utils.cpp index db1754af8a..62cdebb1fa 100644 --- a/src/cpp/ripple/utils.cpp +++ b/src/cpp/ripple/utils.cpp @@ -356,7 +356,7 @@ extern void NameThread(const char* n) { size_t zero = pName.find_first_of('\0'); if ((zero != std::string::npos) && (zero != 0)) - pName = pName.substr(0, zero - 1); + pName = pName.substr(0, zero); size_t slash = pName.find_last_of('/'); if (slash != std::string::npos) pName = pName.substr(slash + 1); From 56fea329dcece960f03eecabc514513c950d9ead Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 05:51:51 -0700 Subject: [PATCH 13/42] Cosmetic --- src/cpp/ripple/LoadManager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/LoadManager.cpp b/src/cpp/ripple/LoadManager.cpp index 56503fff5c..9a09b3378b 100644 --- a/src/cpp/ripple/LoadManager.cpp +++ b/src/cpp/ripple/LoadManager.cpp @@ -317,7 +317,7 @@ int LoadManager::getUptime() void LoadManager::threadEntry() { - NameThread("loadMgr"); + NameThread("loadmgr"); boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time(); while (1) { From 0e34de512e9d9c514191e174871c9fab43ee20c7 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 05:51:59 -0700 Subject: [PATCH 14/42] I like parentheses. --- src/cpp/ripple/Pathfinder.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cpp/ripple/Pathfinder.cpp b/src/cpp/ripple/Pathfinder.cpp index be60e664ed..fd623ae7fb 100644 --- a/src/cpp/ripple/Pathfinder.cpp +++ b/src/cpp/ripple/Pathfinder.cpp @@ -713,9 +713,9 @@ boost::unordered_set usAccountSourceCurrencies(const RippleAddress& raA STAmount saBalance = rspEntry->getBalance(); // Filter out non - if (saBalance.isPositive() // Have IOUs to send. - || (rspEntry->getLimitPeer() // Peer extends credit. - && -saBalance < rspEntry->getLimitPeer())) // Credit left. + if (saBalance.isPositive() // Have IOUs to send. + || (rspEntry->getLimitPeer() // Peer extends credit. + && ((-saBalance) < rspEntry->getLimitPeer()))) // Credit left. { usCurrencies.insert(saBalance.getCurrency()); } From c5429032edf9971d3639ad7546a7f65eeefa77cb Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 06:33:29 -0700 Subject: [PATCH 15/42] Process monitors itself, restarts if it faults, preserves logs and cores. Rate-limiting for safety. --- src/cpp/ripple/Application.cpp | 1 + src/cpp/ripple/main.cpp | 8 ++++ src/cpp/ripple/utils.cpp | 75 ++++++++++++++++++++++++++++++++++ src/cpp/ripple/utils.h | 4 ++ 4 files changed, 88 insertions(+) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 2302c6a631..6c2ea2b178 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -64,6 +64,7 @@ bool Instance::running = true; void Application::stop() { cLog(lsINFO) << "Received shutdown request"; + StopSustain(); mShutdown = true; mIOService.stop(); mHashedObjectStore.bulkWrite(); diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index 320149a036..57aba443e1 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -149,6 +149,7 @@ int main(int argc, char* argv[]) ("ledger", po::value(), "Load the specified ledger and start from .") ("start", "Start from a fresh Ledger.") ("net", "Get the initial ledger from the network.") + ("fg", "Run in the foreground.") ; // Interpret positional arguments as --parameters. @@ -186,6 +187,13 @@ int main(int argc, char* argv[]) } } + if (HaveSustain() && !vm.count("parameters") && !vm.count("fg") && !vm.count("standalone")) + { + std::string logMe = DoSustain(); + if (!logMe.empty()) + Log(lsWARNING) << logMe; + } + if (vm.count("quiet")) Log::setMinSeverity(lsFATAL, true); else if (vm.count("verbose")) diff --git a/src/cpp/ripple/utils.cpp b/src/cpp/ripple/utils.cpp index 62cdebb1fa..6365a862d3 100644 --- a/src/cpp/ripple/utils.cpp +++ b/src/cpp/ripple/utils.cpp @@ -1,6 +1,8 @@ #ifdef __linux__ +#include #include +#include #endif #include @@ -373,6 +375,79 @@ extern void NameThread(const char*) { ; } #endif +#ifdef __unix__ + +static pid_t pManager = static_cast(0); +static pid_t pChild = static_cast(0); + +static void pass_signal(int a) +{ + kill(pChild, a); +} + +static void stop_manager(int) +{ + kill(pChild, SIGINT); + _exit(0); +} + +bool HaveSustain() +{ + return true; +} + +std::string StopSustain() +{ + if (getppid() != pManager) + return std::string(); + kill(pManager, SIGHUP); + return "Terminating monitor"; +} + +std::string DoSustain() +{ + int childCount = 0; + pManager = getpid(); + signal(SIGINT, stop_manager); + signal(SIGHUP, stop_manager); + signal(SIGUSR1, pass_signal); + signal(SIGUSR2, pass_signal); + while (1) + { + ++childCount; + pChild = fork(); + if (pChild == -1) + _exit(0); + if (pChild == 0) + { + NameThread("child"); + signal(SIGINT, SIG_DFL); + signal(SIGHUP, SIG_DFL); + signal(SIGUSR1, SIG_DFL); + signal(SIGUSR2, SIG_DFL); + return str(boost::format("Launching child %d") % childCount);; + } + NameThread(boost::str(boost::format("sust%d") % childCount).c_str()); + do + { + int i; + sleep(10); + waitpid(-1, &i, 0); + } + while (kill(pChild, 0) == 0); + rename("core", boost::str(boost::format("core.%d") % static_cast(pChild)).c_str()); + rename("debug.log", boost::str(boost::format("debug.log.%d") % static_cast(pChild)).c_str()); + } +} + +#else + +bool HaveSustain() { return false; } +std::string DoSustain() { return std::string; } +std::string StopSustain() { return std::string; } + +#endif + BOOST_AUTO_TEST_SUITE( Utils) BOOST_AUTO_TEST_CASE( ParseUrl ) diff --git a/src/cpp/ripple/utils.h b/src/cpp/ripple/utils.h index 8cc6014308..2c60a773f3 100644 --- a/src/cpp/ripple/utils.h +++ b/src/cpp/ripple/utils.h @@ -288,6 +288,10 @@ bool parseUrl(const std::string& strUrl, std::string& strScheme, std::string& st extern void NameThread(const char *); +extern bool HaveSustain(); +extern std::string StopSustain(); +extern std::string DoSustain(); + #if (!defined(FORCE_NO_C11X) && (__cplusplus > 201100L)) || defined(FORCE_C11X) #define C11X From c24c3a315bcc14f9bf2ecbfe0c76d43ca782eb93 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 06:34:29 -0700 Subject: [PATCH 16/42] cosmetic. --- src/cpp/ripple/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/utils.cpp b/src/cpp/ripple/utils.cpp index 6365a862d3..24099ef8d2 100644 --- a/src/cpp/ripple/utils.cpp +++ b/src/cpp/ripple/utils.cpp @@ -427,7 +427,7 @@ std::string DoSustain() signal(SIGUSR2, SIG_DFL); return str(boost::format("Launching child %d") % childCount);; } - NameThread(boost::str(boost::format("sust%d") % childCount).c_str()); + NameThread(boost::str(boost::format("#%d") % childCount).c_str()); do { int i; From 03e7a0ce4072465d3e84cf2f8a684a3242b7a9da Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 06:37:00 -0700 Subject: [PATCH 17/42] Cosmetic. --- src/cpp/ripple/main.cpp | 1 + src/cpp/ripple/utils.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index 57aba443e1..a879bc14fb 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -263,6 +263,7 @@ int main(int argc, char* argv[]) else { // Have a RPC command. + NameThread("rpc"); std::vector vCmd = vm["parameters"].as >(); iResult = commandLineRPC(vCmd); diff --git a/src/cpp/ripple/utils.cpp b/src/cpp/ripple/utils.cpp index 24099ef8d2..a8bbaa75a6 100644 --- a/src/cpp/ripple/utils.cpp +++ b/src/cpp/ripple/utils.cpp @@ -420,7 +420,7 @@ std::string DoSustain() _exit(0); if (pChild == 0) { - NameThread("child"); + NameThread("main"); signal(SIGINT, SIG_DFL); signal(SIGHUP, SIG_DFL); signal(SIGUSR1, SIG_DFL); From 017594950baef67e3bc546b9d1161fa31d5d4f3a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 06:44:00 -0700 Subject: [PATCH 18/42] Use a thunk to name the aux thread. --- src/cpp/ripple/Application.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 6c2ea2b178..7b5dd2191a 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -90,6 +90,12 @@ void sigIntHandler(int) } #endif +static void runAux(boost::asio::io_service& svc) +{ + NameThread("aux"); + svc.run(); +} + void Application::setup() { mJobQueue.setThreadCount(); @@ -117,7 +123,7 @@ void Application::setup() LogPartition::setSeverity(lsDEBUG); } - boost::thread(boost::bind(&boost::asio::io_service::run, &mAuxService)).detach(); + boost::thread(boost::bind(runAux, boost::ref(mAuxService))).detach(); if (!theConfig.RUN_STANDALONE) mSNTPClient.init(theConfig.SNTP_SERVERS); From 92fd82b38e5fa8fc9e2bc6c3bff3fc16124e767b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 06:46:13 -0700 Subject: [PATCH 19/42] Don't relaunch the unit tests. --- src/cpp/ripple/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index a879bc14fb..1b3a9c9df2 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -187,7 +187,8 @@ int main(int argc, char* argv[]) } } - if (HaveSustain() && !vm.count("parameters") && !vm.count("fg") && !vm.count("standalone")) + if (HaveSustain() && + !vm.count("parameters") && !vm.count("fg") && !vm.count("standalone") && !vm.count("unittest")) { std::string logMe = DoSustain(); if (!logMe.empty()) From b829e84c01c496cfba43ffc438abea6811e76725 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 07:07:17 -0700 Subject: [PATCH 20/42] Bump version. --- src/cpp/ripple/Version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/Version.h b/src/cpp/ripple/Version.h index 01ce5a0417..897a19d97e 100644 --- a/src/cpp/ripple/Version.h +++ b/src/cpp/ripple/Version.h @@ -6,7 +6,7 @@ #define SERVER_VERSION_MAJOR 0 #define SERVER_VERSION_MINOR 8 -#define SERVER_VERSION_SUB "-a" +#define SERVER_VERSION_SUB "-b" #define SERVER_NAME "Ripple" #define SV_STRINGIZE(x) SV_STRINGIZE2(x) From 6e9a1f09b56779ab34d46229603d3290e458c9c7 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 08:54:17 -0700 Subject: [PATCH 21/42] Break TransacionAcquire into its own implementation file. --- newcoin.vcxproj | 1 + newcoin.vcxproj.filters | 3 + ripple2010.vcxproj | 1 + ripple2010.vcxproj.filters | 3 + src/cpp/ripple/LedgerConsensus.cpp | 266 ------------------------ src/cpp/ripple/TransactionAcquire.cpp | 287 ++++++++++++++++++++++++++ 6 files changed, 295 insertions(+), 266 deletions(-) create mode 100644 src/cpp/ripple/TransactionAcquire.cpp diff --git a/newcoin.vcxproj b/newcoin.vcxproj index 21c419eb19..fbd79cb56d 100644 --- a/newcoin.vcxproj +++ b/newcoin.vcxproj @@ -174,6 +174,7 @@ + diff --git a/newcoin.vcxproj.filters b/newcoin.vcxproj.filters index e828b851f2..961347ae6f 100644 --- a/newcoin.vcxproj.filters +++ b/newcoin.vcxproj.filters @@ -243,6 +243,9 @@ Source Files + + Source Files + Source Files diff --git a/ripple2010.vcxproj b/ripple2010.vcxproj index e785bc9c3c..843c6f9dcf 100644 --- a/ripple2010.vcxproj +++ b/ripple2010.vcxproj @@ -174,6 +174,7 @@ + diff --git a/ripple2010.vcxproj.filters b/ripple2010.vcxproj.filters index bec881141d..0ef380379d 100644 --- a/ripple2010.vcxproj.filters +++ b/ripple2010.vcxproj.filters @@ -240,6 +240,9 @@ Source Files + + Source Files + Source Files diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index d4fc7d97fa..cabccb7e0a 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -14,8 +14,6 @@ #include "Log.h" #include "SHAMapSync.h" -#define TX_ACQUIRE_TIMEOUT 250 - #define LEDGER_TOTAL_PASSES 8 #define LEDGER_RETRY_PASSES 5 @@ -28,270 +26,6 @@ typedef std::map::value_type u256_lct_pair; SETUP_LOG(); DECLARE_INSTANCE(LedgerConsensus); -DECLARE_INSTANCE(TransactionAcquire); - -TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) -{ - mMap = boost::make_shared(smtTRANSACTION, hash); -} - -void TransactionAcquire::done() -{ - if (mFailed) - { - cLog(lsWARNING) << "Failed to acquire TX set " << mHash; - theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); - } - else - { - cLog(lsINFO) << "Acquired TX set " << mHash; - mMap->setImmutable(); - theApp->getOPs().mapComplete(mHash, mMap); - } - theApp->getMasterLedgerAcquire().dropLedger(mHash); -} - -void TransactionAcquire::onTimer(bool progress) -{ - if (!getPeerCount()) - { // out of peers - cLog(lsWARNING) << "Out of peers for TX set " << getHash(); - - bool found = false; - std::vector peerList = theApp->getConnectionPool().getPeerVector(); - BOOST_FOREACH(Peer::ref peer, peerList) - { - if (peer->hasTxSet(getHash())) - { - found = true; - peerHas(peer); - } - } - if (!found) - { - BOOST_FOREACH(Peer::ref peer, peerList) - peerHas(peer); - } - } - else if (!progress) - trigger(Peer::pointer()); -} - -boost::weak_ptr TransactionAcquire::pmDowncast() -{ - return boost::shared_polymorphic_downcast(shared_from_this()); -} - -void TransactionAcquire::trigger(Peer::ref peer) -{ - if (mComplete || mFailed) - { - cLog(lsINFO) << "complete or failed"; - return; - } - if (!mHaveRoot) - { - cLog(lsTRACE) << "TransactionAcquire::trigger " << (peer ? "havePeer" : "noPeer") << " no root"; - ripple::TMGetLedger tmGL; - tmGL.set_ledgerhash(mHash.begin(), mHash.size()); - tmGL.set_itype(ripple::liTS_CANDIDATE); - if (getTimeouts() != 0) - tmGL.set_querytype(ripple::qtINDIRECT); - *(tmGL.add_nodeids()) = SHAMapNode().getRawString(); - sendRequest(tmGL, peer); - } - else - { - std::vector nodeIDs; - std::vector nodeHashes; - ConsensusTransSetSF sf; - mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &sf); - if (nodeIDs.empty()) - { - if (mMap->isValid()) - mComplete = true; - else - mFailed = true; - done(); - return; - } - ripple::TMGetLedger tmGL; - tmGL.set_ledgerhash(mHash.begin(), mHash.size()); - tmGL.set_itype(ripple::liTS_CANDIDATE); - if (getTimeouts() != 0) - tmGL.set_querytype(ripple::qtINDIRECT); - BOOST_FOREACH(SHAMapNode& it, nodeIDs) - *(tmGL.add_nodeids()) = it.getRawString(); - sendRequest(tmGL, peer); - } -} - -SMAddNode TransactionAcquire::takeNodes(const std::list& nodeIDs, - const std::list< std::vector >& data, Peer::ref peer) -{ - if (mComplete) - { - cLog(lsTRACE) << "TX set complete"; - return SMAddNode(); - } - if (mFailed) - { - cLog(lsTRACE) << "TX set failed"; - return SMAddNode(); - } - try - { - if (nodeIDs.empty()) - return SMAddNode::invalid(); - std::list::const_iterator nodeIDit = nodeIDs.begin(); - std::list< std::vector >::const_iterator nodeDatait = data.begin(); - ConsensusTransSetSF sf; - while (nodeIDit != nodeIDs.end()) - { - if (nodeIDit->isRoot()) - { - if (mHaveRoot) - { - cLog(lsWARNING) << "Got root TXS node, already have it"; - return SMAddNode(); - } - if (!mMap->addRootNode(getHash(), *nodeDatait, snfWIRE, NULL)) - { - cLog(lsWARNING) << "TX acquire got bad root node"; - return SMAddNode::invalid(); - } - else - mHaveRoot = true; - } - else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) - { - cLog(lsWARNING) << "TX acquire got bad non-root node"; - return SMAddNode::invalid(); - } - ++nodeIDit; - ++nodeDatait; - } - trigger(peer); - progress(); - return SMAddNode::useful(); - } - catch (...) - { - cLog(lsERROR) << "Peer sends us junky transaction node data"; - return SMAddNode::invalid(); - } -} - -void LCTransaction::setVote(const uint160& peer, bool votesYes) -{ // Track a peer's yes/no vote on a particular disputed transaction - std::pair::iterator, bool> res = - mVotes.insert(std::pair(peer, votesYes)); - - if (res.second) - { // new vote - if (votesYes) - { - cLog(lsDEBUG) << "Peer " << peer << " votes YES on " << mTransactionID; - ++mYays; - } - else - { - cLog(lsDEBUG) << "Peer " << peer << " votes NO on " << mTransactionID; - ++mNays; - } - } - else if (votesYes && !res.first->second) - { // changes vote to yes - cLog(lsDEBUG) << "Peer " << peer << " now votes YES on " << mTransactionID; - --mNays; - ++mYays; - res.first->second = true; - } - else if (!votesYes && res.first->second) - { // changes vote to no - cLog(lsDEBUG) << "Peer " << peer << " now votes NO on " << mTransactionID; - ++mNays; - --mYays; - res.first->second = false; - } -} - -void LCTransaction::unVote(const uint160& peer) -{ // Remove a peer's vote on this disputed transasction - boost::unordered_map::iterator it = mVotes.find(peer); - if (it != mVotes.end()) - { - if (it->second) - --mYays; - else - --mNays; - mVotes.erase(it); - } -} - -bool LCTransaction::updateVote(int percentTime, bool proposing) -{ - if (mOurVote && (mNays == 0)) - return false; - if (!mOurVote && (mYays == 0)) - return false; - - bool newPosition; - int weight; - if (proposing) // give ourselves full weight - { - // This is basically the percentage of nodes voting 'yes' (including us) - weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); - - // To prevent avalanche stalls, we increase the needed weight slightly over time - if (percentTime < AV_MID_CONSENSUS_TIME) - newPosition = weight > AV_INIT_CONSENSUS_PCT; - else if (percentTime < AV_LATE_CONSENSUS_TIME) - newPosition = weight > AV_MID_CONSENSUS_PCT; - else if (percentTime < AV_STUCK_CONSENSUS_TIME) - newPosition = weight > AV_LATE_CONSENSUS_PCT; - else - newPosition = weight > AV_STUCK_CONSENSUS_PCT; - } - else // don't let us outweigh a proposing node, just recognize consensus - { - weight = -1; - newPosition = mYays > mNays; - } - - if (newPosition == mOurVote) - { - cLog(lsINFO) << - "No change (" << (mOurVote ? "YES" : "NO") << ") : weight " << weight << ", percent " << percentTime; - cLog(lsDEBUG) << getJson(); - return false; - } - - mOurVote = newPosition; - cLog(lsDEBUG) << "We now vote " << (mOurVote ? "YES" : "NO") << " on " << mTransactionID; - cLog(lsDEBUG) << getJson(); - return true; -} - -Json::Value LCTransaction::getJson() -{ - Json::Value ret(Json::objectValue); - - ret["yays"] = mYays; - ret["nays"] = mNays; - ret["our_vote"] = mOurVote; - if (!mVotes.empty()) - { - Json::Value votesj(Json::objectValue); - typedef boost::unordered_map::value_type vt; - BOOST_FOREACH(vt& vote, mVotes) - { - votesj[vote.first.GetHex()] = vote.second; - } - ret["votes"] = votesj; - } - return ret; -} LedgerConsensus::LedgerConsensus(const uint256& prevLCLHash, Ledger::ref previousLedger, uint32 closeTime) : mState(lcsPRE_CLOSE), mCloseTime(closeTime), mPrevLedgerHash(prevLCLHash), mPreviousLedger(previousLedger), diff --git a/src/cpp/ripple/TransactionAcquire.cpp b/src/cpp/ripple/TransactionAcquire.cpp new file mode 100644 index 0000000000..b8e53a7d02 --- /dev/null +++ b/src/cpp/ripple/TransactionAcquire.cpp @@ -0,0 +1,287 @@ +#include "LedgerConsensus.h" + +#include +#include +#include +#include + +#include "../json/writer.h" + +#include "Application.h" +#include "NetworkOPs.h" +#include "LedgerTiming.h" +#include "SerializedValidation.h" +#include "Log.h" +#include "SHAMapSync.h" + +#define TX_ACQUIRE_TIMEOUT 250 + +typedef std::map::value_type u160_prop_pair; +typedef std::map::value_type u256_lct_pair; + +SETUP_LOG(); +DECLARE_INSTANCE(TransactionAcquire); + +TransactionAcquire::TransactionAcquire(const uint256& hash) : PeerSet(hash, TX_ACQUIRE_TIMEOUT), mHaveRoot(false) +{ + mMap = boost::make_shared(smtTRANSACTION, hash); +} + +void TransactionAcquire::done() +{ + if (mFailed) + { + cLog(lsWARNING) << "Failed to acquire TX set " << mHash; + theApp->getOPs().mapComplete(mHash, SHAMap::pointer()); + } + else + { + cLog(lsINFO) << "Acquired TX set " << mHash; + mMap->setImmutable(); + theApp->getOPs().mapComplete(mHash, mMap); + } + theApp->getMasterLedgerAcquire().dropLedger(mHash); +} + +void TransactionAcquire::onTimer(bool progress) +{ + if (!getPeerCount()) + { // out of peers + cLog(lsWARNING) << "Out of peers for TX set " << getHash(); + + bool found = false; + std::vector peerList = theApp->getConnectionPool().getPeerVector(); + BOOST_FOREACH(Peer::ref peer, peerList) + { + if (peer->hasTxSet(getHash())) + { + found = true; + peerHas(peer); + } + } + if (!found) + { + BOOST_FOREACH(Peer::ref peer, peerList) + peerHas(peer); + } + } + else if (!progress) + trigger(Peer::pointer()); +} + +boost::weak_ptr TransactionAcquire::pmDowncast() +{ + return boost::shared_polymorphic_downcast(shared_from_this()); +} + +void TransactionAcquire::trigger(Peer::ref peer) +{ + if (mComplete || mFailed) + { + cLog(lsINFO) << "complete or failed"; + return; + } + if (!mHaveRoot) + { + cLog(lsTRACE) << "TransactionAcquire::trigger " << (peer ? "havePeer" : "noPeer") << " no root"; + ripple::TMGetLedger tmGL; + tmGL.set_ledgerhash(mHash.begin(), mHash.size()); + tmGL.set_itype(ripple::liTS_CANDIDATE); + if (getTimeouts() != 0) + tmGL.set_querytype(ripple::qtINDIRECT); + *(tmGL.add_nodeids()) = SHAMapNode().getRawString(); + sendRequest(tmGL, peer); + } + else + { + std::vector nodeIDs; + std::vector nodeHashes; + ConsensusTransSetSF sf; + mMap->getMissingNodes(nodeIDs, nodeHashes, 256, &sf); + if (nodeIDs.empty()) + { + if (mMap->isValid()) + mComplete = true; + else + mFailed = true; + done(); + return; + } + ripple::TMGetLedger tmGL; + tmGL.set_ledgerhash(mHash.begin(), mHash.size()); + tmGL.set_itype(ripple::liTS_CANDIDATE); + if (getTimeouts() != 0) + tmGL.set_querytype(ripple::qtINDIRECT); + BOOST_FOREACH(SHAMapNode& it, nodeIDs) + *(tmGL.add_nodeids()) = it.getRawString(); + sendRequest(tmGL, peer); + } +} + +SMAddNode TransactionAcquire::takeNodes(const std::list& nodeIDs, + const std::list< std::vector >& data, Peer::ref peer) +{ + if (mComplete) + { + cLog(lsTRACE) << "TX set complete"; + return SMAddNode(); + } + if (mFailed) + { + cLog(lsTRACE) << "TX set failed"; + return SMAddNode(); + } + try + { + if (nodeIDs.empty()) + return SMAddNode::invalid(); + std::list::const_iterator nodeIDit = nodeIDs.begin(); + std::list< std::vector >::const_iterator nodeDatait = data.begin(); + ConsensusTransSetSF sf; + while (nodeIDit != nodeIDs.end()) + { + if (nodeIDit->isRoot()) + { + if (mHaveRoot) + { + cLog(lsWARNING) << "Got root TXS node, already have it"; + return SMAddNode(); + } + if (!mMap->addRootNode(getHash(), *nodeDatait, snfWIRE, NULL)) + { + cLog(lsWARNING) << "TX acquire got bad root node"; + return SMAddNode::invalid(); + } + else + mHaveRoot = true; + } + else if (!mMap->addKnownNode(*nodeIDit, *nodeDatait, &sf)) + { + cLog(lsWARNING) << "TX acquire got bad non-root node"; + return SMAddNode::invalid(); + } + ++nodeIDit; + ++nodeDatait; + } + trigger(peer); + progress(); + return SMAddNode::useful(); + } + catch (...) + { + cLog(lsERROR) << "Peer sends us junky transaction node data"; + return SMAddNode::invalid(); + } +} + +void LCTransaction::setVote(const uint160& peer, bool votesYes) +{ // Track a peer's yes/no vote on a particular disputed transaction + std::pair::iterator, bool> res = + mVotes.insert(std::pair(peer, votesYes)); + + if (res.second) + { // new vote + if (votesYes) + { + cLog(lsDEBUG) << "Peer " << peer << " votes YES on " << mTransactionID; + ++mYays; + } + else + { + cLog(lsDEBUG) << "Peer " << peer << " votes NO on " << mTransactionID; + ++mNays; + } + } + else if (votesYes && !res.first->second) + { // changes vote to yes + cLog(lsDEBUG) << "Peer " << peer << " now votes YES on " << mTransactionID; + --mNays; + ++mYays; + res.first->second = true; + } + else if (!votesYes && res.first->second) + { // changes vote to no + cLog(lsDEBUG) << "Peer " << peer << " now votes NO on " << mTransactionID; + ++mNays; + --mYays; + res.first->second = false; + } +} + +void LCTransaction::unVote(const uint160& peer) +{ // Remove a peer's vote on this disputed transasction + boost::unordered_map::iterator it = mVotes.find(peer); + if (it != mVotes.end()) + { + if (it->second) + --mYays; + else + --mNays; + mVotes.erase(it); + } +} + +bool LCTransaction::updateVote(int percentTime, bool proposing) +{ + if (mOurVote && (mNays == 0)) + return false; + if (!mOurVote && (mYays == 0)) + return false; + + bool newPosition; + int weight; + if (proposing) // give ourselves full weight + { + // This is basically the percentage of nodes voting 'yes' (including us) + weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); + + // To prevent avalanche stalls, we increase the needed weight slightly over time + if (percentTime < AV_MID_CONSENSUS_TIME) + newPosition = weight > AV_INIT_CONSENSUS_PCT; + else if (percentTime < AV_LATE_CONSENSUS_TIME) + newPosition = weight > AV_MID_CONSENSUS_PCT; + else if (percentTime < AV_STUCK_CONSENSUS_TIME) + newPosition = weight > AV_LATE_CONSENSUS_PCT; + else + newPosition = weight > AV_STUCK_CONSENSUS_PCT; + } + else // don't let us outweigh a proposing node, just recognize consensus + { + weight = -1; + newPosition = mYays > mNays; + } + + if (newPosition == mOurVote) + { + cLog(lsINFO) << + "No change (" << (mOurVote ? "YES" : "NO") << ") : weight " << weight << ", percent " << percentTime; + cLog(lsDEBUG) << getJson(); + return false; + } + + mOurVote = newPosition; + cLog(lsDEBUG) << "We now vote " << (mOurVote ? "YES" : "NO") << " on " << mTransactionID; + cLog(lsDEBUG) << getJson(); + return true; +} + +Json::Value LCTransaction::getJson() +{ + Json::Value ret(Json::objectValue); + + ret["yays"] = mYays; + ret["nays"] = mNays; + ret["our_vote"] = mOurVote; + if (!mVotes.empty()) + { + Json::Value votesj(Json::objectValue); + typedef boost::unordered_map::value_type vt; + BOOST_FOREACH(vt& vote, mVotes) + { + votesj[vote.first.GetHex()] = vote.second; + } + ret["votes"] = votesj; + } + return ret; +} + From 72f291edd1814a66e2e767890c503336ed5c8df1 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 08:54:39 -0700 Subject: [PATCH 22/42] Better logging of connections to cluster peers. --- src/cpp/ripple/Peer.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index ee0077beb7..9dbd04421b 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -95,6 +95,8 @@ void Peer::detach(const char *rsn, bool onIOStrand) if (!mDetaching) { mDetaching = true; // Race is ok. + + tLog(mCluster, lsWARNING) << "Cluster peer detach \"" << mNodeName << "\": " << rsn; /* cLog(lsDEBUG) << "Peer: Detach: " << ADDRESS(this) << "> " @@ -384,7 +386,15 @@ void Peer::handleReadHeader(const boost::system::error_code& error) } else { - cLog(lsINFO) << "Peer: Header: Error: " << getIP() << ": " << error.category().name() << ": " << error.message() << ": " << error; + if (mCluster) + { + cLog(lsINFO) << "Peer: Cluster connection lost to \"" << mNodeName << "\": " << + error.category().name() << ": " << error.message() << ": " << error; + } + else + { + cLog(lsINFO) << "Peer: Header: Error: " << getIP() << ": " << error.category().name() << ": " << error.message() << ": " << error; + } detach("hrh2", true); } } @@ -397,7 +407,15 @@ void Peer::handleReadBody(const boost::system::error_code& error) } else if (error) { - cLog(lsINFO) << "Peer: Body: Error: " << ADDRESS(this) << ": " << error.category().name() << ": " << error.message() << ": " << error; + if (mCluster) + { + cLog(lsINFO) << "Peer: Cluster connection lost to \"" << mNodeName << "\": " << + error.category().name() << ": " << error.message() << ": " << error; + } + else + { + cLog(lsINFO) << "Peer: Body: Error: " << getIP() << ": " << error.category().name() << ": " << error.message() << ": " << error; + } boost::recursive_mutex::scoped_lock sl(theApp->getMasterLock()); detach("hrb", true); return; @@ -734,6 +752,8 @@ void Peer::recvHello(ripple::TMHello& packet) { mCluster = true; mLoad.setPrivileged(); + cLog(lsINFO) << "Cluster connection to \"" << (mNodeName.empty() ? getIP() : mNodeName) + << "\" established"; } if (isOutbound()) mLoad.setOutbound(); From d8a384467af9cb005eb1ead47fe32b8dfa2fa853 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 08:58:19 -0700 Subject: [PATCH 23/42] Break out some ConsensusTransSetSF functions so I can work on them. --- src/cpp/ripple/SHAMapSync.h | 14 ++++---------- src/cpp/ripple/TransactionAcquire.cpp | 13 +++++++++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/cpp/ripple/SHAMapSync.h b/src/cpp/ripple/SHAMapSync.h index 7c3eae3455..35b6e9b794 100644 --- a/src/cpp/ripple/SHAMapSync.h +++ b/src/cpp/ripple/SHAMapSync.h @@ -12,17 +12,11 @@ class ConsensusTransSetSF : public SHAMapSyncFilter { // sync filter for transaction sets during consensus building public: ConsensusTransSetSF() { ; } + virtual void gotNode(const SHAMapNode& id, const uint256& nodeHash, - const std::vector& nodeData, SHAMapTreeNode::TNType) - { - // WRITEME: If 'isLeaf' is true, this is a transaction - theApp->getTempNodeCache().store(nodeHash, nodeData); - } - virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) - { - // WRITEME: We could check our own map, we could check transaction tables - return theApp->getTempNodeCache().retrieve(nodeHash, nodeData); - } + const std::vector& nodeData, SHAMapTreeNode::TNType); + + virtual bool haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData); }; // This class is only needed on add functions diff --git a/src/cpp/ripple/TransactionAcquire.cpp b/src/cpp/ripple/TransactionAcquire.cpp index b8e53a7d02..3a5155f7a6 100644 --- a/src/cpp/ripple/TransactionAcquire.cpp +++ b/src/cpp/ripple/TransactionAcquire.cpp @@ -285,3 +285,16 @@ Json::Value LCTransaction::getJson() return ret; } +void ConsensusTransSetSF::gotNode(const SHAMapNode& id, const uint256& nodeHash, + const std::vector& nodeData, SHAMapTreeNode::TNType) +{ + // WRITEME: If 'isLeaf' is true, this is a transaction + theApp->getTempNodeCache().store(nodeHash, nodeData); +} + +bool ConsensusTransSetSF::haveNode(const SHAMapNode& id, const uint256& nodeHash, + std::vector& nodeData) +{ + // WRITEME: We could check our own map, we could check transaction tables + return theApp->getTempNodeCache().retrieve(nodeHash, nodeData); +} From 371d52cc9ec32dab73c4fbc2b502878c6c74febf Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 09:22:30 -0700 Subject: [PATCH 24/42] Move LCT code to the right file. Be smarter in the consensus set filter code. --- src/cpp/ripple/LedgerConsensus.cpp | 111 +++++++++++++++++++ src/cpp/ripple/Transaction.h | 2 +- src/cpp/ripple/TransactionAcquire.cpp | 151 ++++++-------------------- 3 files changed, 147 insertions(+), 117 deletions(-) diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index cabccb7e0a..971bb332ae 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -27,6 +27,117 @@ typedef std::map::value_type u256_lct_pair; SETUP_LOG(); DECLARE_INSTANCE(LedgerConsensus); +void LCTransaction::setVote(const uint160& peer, bool votesYes) +{ // Track a peer's yes/no vote on a particular disputed transaction + std::pair::iterator, bool> res = + mVotes.insert(std::pair(peer, votesYes)); + + if (res.second) + { // new vote + if (votesYes) + { + cLog(lsDEBUG) << "Peer " << peer << " votes YES on " << mTransactionID; + ++mYays; + } + else + { + cLog(lsDEBUG) << "Peer " << peer << " votes NO on " << mTransactionID; + ++mNays; + } + } + else if (votesYes && !res.first->second) + { // changes vote to yes + cLog(lsDEBUG) << "Peer " << peer << " now votes YES on " << mTransactionID; + --mNays; + ++mYays; + res.first->second = true; + } + else if (!votesYes && res.first->second) + { // changes vote to no + cLog(lsDEBUG) << "Peer " << peer << " now votes NO on " << mTransactionID; + ++mNays; + --mYays; + res.first->second = false; + } +} + +void LCTransaction::unVote(const uint160& peer) +{ // Remove a peer's vote on this disputed transasction + boost::unordered_map::iterator it = mVotes.find(peer); + if (it != mVotes.end()) + { + if (it->second) + --mYays; + else + --mNays; + mVotes.erase(it); + } +} + +bool LCTransaction::updateVote(int percentTime, bool proposing) +{ + if (mOurVote && (mNays == 0)) + return false; + if (!mOurVote && (mYays == 0)) + return false; + + bool newPosition; + int weight; + if (proposing) // give ourselves full weight + { + // This is basically the percentage of nodes voting 'yes' (including us) + weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); + + // To prevent avalanche stalls, we increase the needed weight slightly over time + if (percentTime < AV_MID_CONSENSUS_TIME) + newPosition = weight > AV_INIT_CONSENSUS_PCT; + else if (percentTime < AV_LATE_CONSENSUS_TIME) + newPosition = weight > AV_MID_CONSENSUS_PCT; + else if (percentTime < AV_STUCK_CONSENSUS_TIME) + newPosition = weight > AV_LATE_CONSENSUS_PCT; + else + newPosition = weight > AV_STUCK_CONSENSUS_PCT; + } + else // don't let us outweigh a proposing node, just recognize consensus + { + weight = -1; + newPosition = mYays > mNays; + } + + if (newPosition == mOurVote) + { + cLog(lsINFO) << + "No change (" << (mOurVote ? "YES" : "NO") << ") : weight " << weight << ", percent " << percentTime; + cLog(lsDEBUG) << getJson(); + return false; + } + + mOurVote = newPosition; + cLog(lsDEBUG) << "We now vote " << (mOurVote ? "YES" : "NO") << " on " << mTransactionID; + cLog(lsDEBUG) << getJson(); + return true; +} + +Json::Value LCTransaction::getJson() +{ + Json::Value ret(Json::objectValue); + + ret["yays"] = mYays; + ret["nays"] = mNays; + ret["our_vote"] = mOurVote; + if (!mVotes.empty()) + { + Json::Value votesj(Json::objectValue); + typedef boost::unordered_map::value_type vt; + BOOST_FOREACH(vt& vote, mVotes) + { + votesj[vote.first.GetHex()] = vote.second; + } + ret["votes"] = votesj; + } + return ret; +} + LedgerConsensus::LedgerConsensus(const uint256& prevLCLHash, Ledger::ref previousLedger, uint32 closeTime) : mState(lcsPRE_CLOSE), mCloseTime(closeTime), mPrevLedgerHash(prevLCLHash), mPreviousLedger(previousLedger), mValPublic(theConfig.VALIDATION_PUB), mValPrivate(theConfig.VALIDATION_PRIV), mConsensusFail(false), diff --git a/src/cpp/ripple/Transaction.h b/src/cpp/ripple/Transaction.h index 0ec8eadba9..cc4a4bc77b 100644 --- a/src/cpp/ripple/Transaction.h +++ b/src/cpp/ripple/Transaction.h @@ -77,7 +77,7 @@ public: bool checkSign() const; void updateID() { mTransactionID=mTransaction->getTransactionID(); } - SerializedTransaction::pointer getSTransaction() { return mTransaction; } + SerializedTransaction::ref getSTransaction() { return mTransaction; } const uint256& getID() const { return mTransactionID; } const RippleAddress& getFromAccount() const { return mAccountFrom; } diff --git a/src/cpp/ripple/TransactionAcquire.cpp b/src/cpp/ripple/TransactionAcquire.cpp index 3a5155f7a6..69630d4d55 100644 --- a/src/cpp/ripple/TransactionAcquire.cpp +++ b/src/cpp/ripple/TransactionAcquire.cpp @@ -9,10 +9,10 @@ #include "Application.h" #include "NetworkOPs.h" -#include "LedgerTiming.h" #include "SerializedValidation.h" #include "Log.h" #include "SHAMapSync.h" +#include "HashPrefixes.h" #define TX_ACQUIRE_TIMEOUT 250 @@ -174,127 +174,46 @@ SMAddNode TransactionAcquire::takeNodes(const std::list& nodeIDs, } } -void LCTransaction::setVote(const uint160& peer, bool votesYes) -{ // Track a peer's yes/no vote on a particular disputed transaction - std::pair::iterator, bool> res = - mVotes.insert(std::pair(peer, votesYes)); - - if (res.second) - { // new vote - if (votesYes) - { - cLog(lsDEBUG) << "Peer " << peer << " votes YES on " << mTransactionID; - ++mYays; - } - else - { - cLog(lsDEBUG) << "Peer " << peer << " votes NO on " << mTransactionID; - ++mNays; - } - } - else if (votesYes && !res.first->second) - { // changes vote to yes - cLog(lsDEBUG) << "Peer " << peer << " now votes YES on " << mTransactionID; - --mNays; - ++mYays; - res.first->second = true; - } - else if (!votesYes && res.first->second) - { // changes vote to no - cLog(lsDEBUG) << "Peer " << peer << " now votes NO on " << mTransactionID; - ++mNays; - --mYays; - res.first->second = false; - } -} - -void LCTransaction::unVote(const uint160& peer) -{ // Remove a peer's vote on this disputed transasction - boost::unordered_map::iterator it = mVotes.find(peer); - if (it != mVotes.end()) - { - if (it->second) - --mYays; - else - --mNays; - mVotes.erase(it); - } -} - -bool LCTransaction::updateVote(int percentTime, bool proposing) -{ - if (mOurVote && (mNays == 0)) - return false; - if (!mOurVote && (mYays == 0)) - return false; - - bool newPosition; - int weight; - if (proposing) // give ourselves full weight - { - // This is basically the percentage of nodes voting 'yes' (including us) - weight = (mYays * 100 + (mOurVote ? 100 : 0)) / (mNays + mYays + 1); - - // To prevent avalanche stalls, we increase the needed weight slightly over time - if (percentTime < AV_MID_CONSENSUS_TIME) - newPosition = weight > AV_INIT_CONSENSUS_PCT; - else if (percentTime < AV_LATE_CONSENSUS_TIME) - newPosition = weight > AV_MID_CONSENSUS_PCT; - else if (percentTime < AV_STUCK_CONSENSUS_TIME) - newPosition = weight > AV_LATE_CONSENSUS_PCT; - else - newPosition = weight > AV_STUCK_CONSENSUS_PCT; - } - else // don't let us outweigh a proposing node, just recognize consensus - { - weight = -1; - newPosition = mYays > mNays; - } - - if (newPosition == mOurVote) - { - cLog(lsINFO) << - "No change (" << (mOurVote ? "YES" : "NO") << ") : weight " << weight << ", percent " << percentTime; - cLog(lsDEBUG) << getJson(); - return false; - } - - mOurVote = newPosition; - cLog(lsDEBUG) << "We now vote " << (mOurVote ? "YES" : "NO") << " on " << mTransactionID; - cLog(lsDEBUG) << getJson(); - return true; -} - -Json::Value LCTransaction::getJson() -{ - Json::Value ret(Json::objectValue); - - ret["yays"] = mYays; - ret["nays"] = mNays; - ret["our_vote"] = mOurVote; - if (!mVotes.empty()) - { - Json::Value votesj(Json::objectValue); - typedef boost::unordered_map::value_type vt; - BOOST_FOREACH(vt& vote, mVotes) - { - votesj[vote.first.GetHex()] = vote.second; - } - ret["votes"] = votesj; - } - return ret; -} - void ConsensusTransSetSF::gotNode(const SHAMapNode& id, const uint256& nodeHash, - const std::vector& nodeData, SHAMapTreeNode::TNType) + const std::vector& nodeData, SHAMapTreeNode::TNType type) { - // WRITEME: If 'isLeaf' is true, this is a transaction theApp->getTempNodeCache().store(nodeHash, nodeData); + if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size() > 16)) + { // this is a transaction, and we didn't have it + cLog(lsDEBUG) << "Node on our acquiring TX set is TXN we don't have"; + try + { + Serializer s(nodeData.begin() + 4, nodeData.end()); // skip prefix + SerializerIterator sit(s); + SerializedTransaction::pointer stx = boost::make_shared(boost::ref(sit)); + assert(stx->getTransactionID() == nodeHash); + theApp->getJobQueue().addJob(jtTRANSACTION, "TXS->TXN", + BIND_TYPE(&NetworkOPs::submitTransaction, &theApp->getOPs(), P_1, stx, NetworkOPs::stCallback())); + } + catch (...) + { + cLog(lsWARNING) << "Fetched invalid transaction in proposed set"; + } + } } bool ConsensusTransSetSF::haveNode(const SHAMapNode& id, const uint256& nodeHash, std::vector& nodeData) { - // WRITEME: We could check our own map, we could check transaction tables - return theApp->getTempNodeCache().retrieve(nodeHash, nodeData); + if (theApp->getTempNodeCache().retrieve(nodeHash, nodeData)) + return true; + + Transaction::pointer txn = Transaction::load(nodeHash); + if (txn) + { // this is a transaction, and we have it + cLog(lsDEBUG) << "Node in our acquiring TX set is TXN we have"; + Serializer s; + s.add32(sHP_TransactionID); + txn->getSTransaction()->add(s, true); + assert(s.getSHA512Half() == nodeHash); + nodeData = s.peekData(); + return true; + } + + return false; } From 9ff59c93f8250814c672108e0b7e717e59abdbbe Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 09:31:03 -0700 Subject: [PATCH 25/42] Fix bogus comment and race condition in ledger fetch code. --- src/cpp/ripple/Peer.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index 9dbd04421b..5f26291316 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -1414,8 +1414,7 @@ void Peer::recvGetLedger(ripple::TMGetLedger& packet) if (!map) { if (packet.has_querytype() && !packet.has_requestcookie()) - { // FIXME: Don't relay requests for older ledgers we would acquire - // (if we don't have them, we can't get them) + { cLog(lsDEBUG) << "Trying to route TX set request"; std::vector peerList = theApp->getConnectionPool().getPeerVector(); std::vector usablePeers; @@ -1642,7 +1641,7 @@ void Peer::recvLedger(const boost::shared_ptr& packet_ptr) if (target) { packet.clear_requestcookie(); - target->sendPacket(boost::make_shared(packet, ripple::mtLEDGER_DATA), true); + target->sendPacket(boost::make_shared(packet, ripple::mtLEDGER_DATA), false); } else { From c961ae581554448a70efa763a2b031780ee05b79 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 16:53:39 -0700 Subject: [PATCH 26/42] Ensure stable order in account tx responses. --- src/cpp/ripple/NetworkOPs.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index db8a28c0d5..26a8da1bff 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1068,7 +1068,7 @@ std::vector< std::pair > str(boost::format("SELECT AccountTransactions.LedgerSeq,Status,RawTxn,TxnMeta FROM " "AccountTransactions INNER JOIN Transactions ON Transactions.TransID = AccountTransactions.TransID " "WHERE Account = '%s' AND AccountTransactions.LedgerSeq <= '%u' AND AccountTransactions.LedgerSeq >= '%u' " - "ORDER BY AccountTransactions.LedgerSeq DESC LIMIT 200;") + "ORDER BY AccountTransactions.LedgerSeq,AccountTransactions.TransID DESC LIMIT 200;") % account.humanAccountID() % maxLedger % minLedger); { @@ -1105,7 +1105,7 @@ std::vector NetworkOPs::getAccountTxsB( std::string sql = str(boost::format("SELECT AccountTransactions.LedgerSeq,Status,RawTxn,TxnMeta FROM " "AccountTransactions INNER JOIN Transactions ON Transactions.TransID = AccountTransactions.TransID " "WHERE Account = '%s' AND AccountTransactions.LedgerSeq <= '%u' AND AccountTransactions.LedgerSeq >= '%u' " - "ORDER BY AccountTransactions.LedgerSeq DESC LIMIT 500;") + "ORDER BY AccountTransactions.LedgerSeq,AccountTransactions.TransID DESC LIMIT 500;") % account.humanAccountID() % maxLedger % minLedger); { From 98754ee962a400b942cdabf187b3ececda693c99 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 18:28:19 -0700 Subject: [PATCH 27/42] Round XRP less aggressively. --- src/cpp/ripple/AmountRound.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/AmountRound.cpp b/src/cpp/ripple/AmountRound.cpp index 10f7d7b211..2dd9eb874d 100644 --- a/src/cpp/ripple/AmountRound.cpp +++ b/src/cpp/ripple/AmountRound.cpp @@ -32,26 +32,31 @@ static void canonicalizeRound(bool isNative, uint64& value, int& offset, bool ro { if (offset < 0) { + int loops = 0; while (offset < -1) { value /= 10; ++offset; + ++loops; } - value += 10; // add before last divide + value += (loops >= 2) ? 9 : 10; // add before last divide value /= 10; ++offset; } } else if (value > STAmount::cMaxValue) { + cLog(lsWARNING) << "A: " << value << " : " << offset; while (value > (10 * STAmount::cMaxValue)) { value /= 10; ++offset; } + cLog(lsWARNING) << "B: " << value << " : " << offset; value += 9; // add before last divide value /= 10; ++offset; + cLog(lsWARNING) << "C: " << value << " : " << offset; } cLog(lsDEBUG) << "canonicalize> " << value << ":" << offset << (roundUp ? " up" : " down"); } @@ -294,6 +299,10 @@ BOOST_AUTO_TEST_SUITE(amountRound) BOOST_AUTO_TEST_CASE( amountRound_test ) { + uint64 value = 25000000000000000ull; + int offset = -14; + canonicalizeRound(false, value, offset, true); + STAmount one(CURRENCY_ONE, ACCOUNT_ONE, 1); STAmount two(CURRENCY_ONE, ACCOUNT_ONE, 2); STAmount three(CURRENCY_ONE, ACCOUNT_ONE, 3); From 3943929fe3f09781d0a1408717f91146157cfd8b Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 18:29:03 -0700 Subject: [PATCH 28/42] Remove inadvertant debug. --- src/cpp/ripple/AmountRound.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/cpp/ripple/AmountRound.cpp b/src/cpp/ripple/AmountRound.cpp index 2dd9eb874d..12deef7c9a 100644 --- a/src/cpp/ripple/AmountRound.cpp +++ b/src/cpp/ripple/AmountRound.cpp @@ -46,17 +46,14 @@ static void canonicalizeRound(bool isNative, uint64& value, int& offset, bool ro } else if (value > STAmount::cMaxValue) { - cLog(lsWARNING) << "A: " << value << " : " << offset; while (value > (10 * STAmount::cMaxValue)) { value /= 10; ++offset; } - cLog(lsWARNING) << "B: " << value << " : " << offset; value += 9; // add before last divide value /= 10; ++offset; - cLog(lsWARNING) << "C: " << value << " : " << offset; } cLog(lsDEBUG) << "canonicalize> " << value << ":" << offset << (roundUp ? " up" : " down"); } From 837de8a7a70aab167de62927429cde8d30512043 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 20 Mar 2013 18:32:34 -0700 Subject: [PATCH 29/42] Fix ripple through micro offer reverse. --- src/cpp/ripple/RippleCalc.cpp | 27 +++++++++++++++++++-------- test/offer-test.js | 5 +++-- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/cpp/ripple/RippleCalc.cpp b/src/cpp/ripple/RippleCalc.cpp index b8a0b0f0ed..993783f681 100644 --- a/src/cpp/ripple/RippleCalc.cpp +++ b/src/cpp/ripple/RippleCalc.cpp @@ -788,7 +788,7 @@ Json::Value PathState::getJson() const // If needed, advance to next funded offer. // - Automatically advances to first offer. -// - Set bEntryAdvance to advance to next entry. +// --> bEntryAdvance: true, to advance to next entry. false, recalculate. // <-- uOfferIndex : 0=end of list. TER RippleCalc::calcNodeAdvance( const unsigned int uNode, // 0 < uNode < uLast @@ -1081,8 +1081,8 @@ TER RippleCalc::calcNodeAdvance( } // Between offer nodes, the fee charged may vary. Therefore, process one inbound offer at a time. Propagate the inbound offer's -// requirements to the previous node. The previous node adjusts the amount output and the amount spent on fees. Continue process -// till request is satisified while we the rate does not increase past the initial rate. +// requirements to the previous node. The previous node adjusts the amount output and the amount spent on fees. Continue +// processing until the request is satisified as long as the rate does not increase past the initial rate. TER RippleCalc::calcNodeDeliverRev( const unsigned int uNode, // 0 < uNode < uLast PathState& psCur, @@ -1171,6 +1171,9 @@ TER RippleCalc::calcNodeDeliverRev( else if (saOutFeeRate < saRateMax) { // Reducing rate. Additional offers will only considered for this increment if they are at least this good. + // At this point, the overall rate is reducing, while the overall rate is not saOutFeeRate, it would be wrong to add + // anthing with a rate above saOutFeeRate. + // The rate would be reduced if the current offer was from the issuer and the previous offer wasn't. saRateMax = saOutFeeRate; @@ -1207,8 +1210,7 @@ TER RippleCalc::calcNodeDeliverRev( // Compute portion of input needed to cover actual output. - // XXX This needs to round up! - STAmount saInPassReq = STAmount::multiply(saOutPass, saOfrRate, saTakerPays); + STAmount saInPassReq = STAmount::mulRound(saOutPass, saOfrRate, saTakerPays, true); STAmount saInPassAct; cLog(lsINFO) << boost::str(boost::format("calcNodeDeliverRev: saInPassReq=%s saOfrRate=%s saOutPass=%s saOutPlusFees=%s") @@ -1217,8 +1219,16 @@ TER RippleCalc::calcNodeDeliverRev( % saOutPass % saOutPlusFees); + if (!saInPassReq) + { + // After rounding did not want anything. + cLog(lsINFO) << boost::str(boost::format("calcNodeDeliverRev: micro offer is unfunded.")); + + bEntryAdvance = true; + continue; + } // Find out input amount actually available at current rate. - if (!!uPrvAccountID) + else if (!!uPrvAccountID) { // account --> OFFER --> ? // Due to node expansion, previous is guaranteed to be the issuer. @@ -1300,15 +1310,16 @@ TER RippleCalc::calcNodeDeliverRev( // Offer became unfunded. cLog(lsINFO) << boost::str(boost::format("calcNodeDeliverRev: offer became unfunded.")); - bEntryAdvance = true; + bEntryAdvance = true; // XXX When don't we want to set advance? } saOutAct += saOutPass; saPrvDlvReq += saInPassAct; } +// XXX Perhaps need to check if partial is okay to relax this? if (tesSUCCESS == terResult && !saOutAct) - terResult = tecPATH_DRY; + terResult = tecPATH_DRY; // Unable to meet request, consider path dry. return terResult; } diff --git a/test/offer-test.js b/test/offer-test.js index dd7a62924a..13724f6f5f 100644 --- a/test/offer-test.js +++ b/test/offer-test.js @@ -1072,6 +1072,7 @@ buster.testCase("Offer tests", { buster.testCase("Offer cross currency", { 'setUp' : testutils.build_setup(), + // 'setUp' : testutils.build_setup({ verbose: true }), 'tearDown' : testutils.build_teardown(), "ripple cross currency payment - start with XRP" : @@ -1112,7 +1113,7 @@ buster.testCase("Offer cross currency", { self.what = "Create offer."; self.remote.transaction() - .offer_create("carol", "500", "50/USD/mtgox") + .offer_create("carol", "500.0", "50/USD/mtgox") .on('proposed', function (m) { // console.log("PROPOSED: offer_create: %s", JSON.stringify(m)); callback(m.result !== 'tesSUCCESS'); @@ -1126,7 +1127,7 @@ buster.testCase("Offer cross currency", { self.remote.transaction() .payment("alice", "bob", "25/USD/mtgox") - .send_max("333") + .send_max("333.0") .on('proposed', function (m) { // console.log("proposed: %s", JSON.stringify(m)); From 8a6c88f5ff06e8395fcb6eb0b9bf2048448c6f75 Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 20 Mar 2013 20:10:34 -0700 Subject: [PATCH 30/42] Fix divRound. --- src/cpp/ripple/SerializedTypes.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/SerializedTypes.h b/src/cpp/ripple/SerializedTypes.h index 96306a79f2..562886bfe0 100644 --- a/src/cpp/ripple/SerializedTypes.h +++ b/src/cpp/ripple/SerializedTypes.h @@ -442,7 +442,7 @@ public: static STAmount divRound(const STAmount& v1, const STAmount& v2, const STAmount& saUnit, bool roundUp) { return divRound(v1, v2, saUnit.getCurrency(), saUnit.getIssuer(), roundUp); } static STAmount divRound(const STAmount& v1, const STAmount& v2, bool roundUp) - { return divRound(v1, v2, v2.getCurrency(), v2.getIssuer(), roundUp); } + { return divRound(v1, v2, v1.getCurrency(), v1.getIssuer(), roundUp); } // Someone is offering X for Y, what is the rate? // Rate: smaller is better, the taker wants the most out: in/out From 7074842fbc06293e933f629327986bc70f4f9adb Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Wed, 20 Mar 2013 20:10:57 -0700 Subject: [PATCH 31/42] Improve ripple rounding. --- src/cpp/ripple/RippleCalc.cpp | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/cpp/ripple/RippleCalc.cpp b/src/cpp/ripple/RippleCalc.cpp index 993783f681..bf94a47839 100644 --- a/src/cpp/ripple/RippleCalc.cpp +++ b/src/cpp/ripple/RippleCalc.cpp @@ -708,6 +708,7 @@ void RippleCalc::setCanonical(STPathSet& spsDst, const std::vector uQualityOut // saPrvReq -> saCurReq // sqPrvAct -> saCurAct @@ -1728,7 +1732,7 @@ void RippleCalc::calcNodeRipple( const uint160 uCurIssuerID = saCur.getIssuer(); // const uint160 uPrvIssuerID = saPrv.getIssuer(); - STAmount saCurIn = STAmount::divide(STAmount::multiply(saCur, uQualityOut, uCurrencyID, uCurIssuerID), uQualityIn, uCurrencyID, uCurIssuerID); + STAmount saCurIn = STAmount::divRound(STAmount::mulRound(saCur, uQualityOut, uCurrencyID, uCurIssuerID, true), uQualityIn, uCurrencyID, uCurIssuerID, true); cLog(lsTRACE) << boost::str(boost::format("calcNodeRipple: bPrvUnlimited=%d saPrv=%s saCurIn=%s") % bPrvUnlimited % saPrv.getFullText() % saCurIn.getFullText()); if (bPrvUnlimited || saCurIn <= saPrv) @@ -1741,7 +1745,7 @@ void RippleCalc::calcNodeRipple( else { // A part of cur. All of prv. (cur as driver) - STAmount saCurOut = STAmount::divide(STAmount::multiply(saPrv, uQualityIn, uCurrencyID, uCurIssuerID), uQualityOut, uCurrencyID, uCurIssuerID); + STAmount saCurOut = STAmount::divRound(STAmount::mulRound(saPrv, uQualityIn, uCurrencyID, uCurIssuerID, true), uQualityOut, uCurrencyID, uCurIssuerID, true); cLog(lsTRACE) << boost::str(boost::format("calcNodeRipple:4: saCurReq=%s") % saCurReq.getFullText()); saCurAct += saCurOut; @@ -2239,7 +2243,7 @@ TER RippleCalc::calcNodeAccountFwd( STAmount saIssueCrd = uQualityIn >= QUALITY_ONE ? saPrvIssueReq // No fee. - : STAmount::multiply(saPrvIssueReq, STAmount(CURRENCY_ONE, ACCOUNT_ONE, uQualityIn, -9)); // Amount to credit. + : STAmount::mulRound(saPrvIssueReq, STAmount(CURRENCY_ONE, ACCOUNT_ONE, uQualityIn, -9), false); // Amount to credit. // Amount to credit. Credit for less than received as a surcharge. saCurReceive = saPrvRedeemReq+saIssueCrd; From ec4118a25a2fad7c44924d9419228fa604bc15e1 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Wed, 20 Mar 2013 22:56:19 -0700 Subject: [PATCH 32/42] More debug of some validations to make sure suppression is working right. --- src/cpp/ripple/ValidationCollection.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/ValidationCollection.cpp b/src/cpp/ripple/ValidationCollection.cpp index a3bdb951bb..2827c36f52 100644 --- a/src/cpp/ripple/ValidationCollection.cpp +++ b/src/cpp/ripple/ValidationCollection.cpp @@ -52,7 +52,8 @@ bool ValidationCollection::addValidation(SerializedValidation::ref val) } else { - cLog(lsINFO) << "Node " << signer.humanNodePublic() << " not in UNL"; + cLog(lsDEBUG) << "Node " << signer.humanNodePublic() << " not in UNL st=" << val->getSignTime() << + ", hash=" << val->getLedgerHash() << ", shash=" << val->getSigningHash(); } uint256 hash = val->getLedgerHash(); From 9b20a3ae71281afb6123d9d729c95c32f24a2662 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 21 Mar 2013 06:02:46 -0700 Subject: [PATCH 33/42] Don't scan entire account transaction database when adding transaction. --- src/cpp/ripple/DBInit.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/cpp/ripple/DBInit.cpp b/src/cpp/ripple/DBInit.cpp index 4e2884b23d..1ae2f7efd0 100644 --- a/src/cpp/ripple/DBInit.cpp +++ b/src/cpp/ripple/DBInit.cpp @@ -24,6 +24,8 @@ const char *TxnDBInit[] = { Account CHARACTER(64), \ LedgerSeq BIGINT UNSIGNED \ );", + "CREATE INDEX AcctTxIDIndex ON \ + AccountTransactions(TransID);", "CREATE INDEX AcctTxIndex ON \ AccountTransactions(Account, LedgerSeq, TransID);", "CREATE INDEX AcctLgrIndex ON \ From b0c694195c542ad655be3e9f14969d76f1dcbb0a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 21 Mar 2013 06:03:17 -0700 Subject: [PATCH 34/42] Add 'getDisplayName' file. --- src/cpp/ripple/Peer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cpp/ripple/Peer.h b/src/cpp/ripple/Peer.h index 1244a5170f..36ba93714e 100644 --- a/src/cpp/ripple/Peer.h +++ b/src/cpp/ripple/Peer.h @@ -117,6 +117,7 @@ public: //bool operator == (const Peer& other); std::string& getIP() { return mIpPort.first; } + std::string getDisplayName() { return mCluster ? mNodeName : mIpPort.first; } int getPort() { return mIpPort.second; } void setIpPort(const std::string& strIP, int iPort); From cd14be1afb9d4207ba993400a08e0038a71f615a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Thu, 21 Mar 2013 06:03:47 -0700 Subject: [PATCH 35/42] Better logging of untrusted validations. (To make sure there's no bug.) --- src/cpp/ripple/LedgerConsensus.cpp | 4 ++-- src/cpp/ripple/NetworkOPs.cpp | 6 +++--- src/cpp/ripple/NetworkOPs.h | 2 +- src/cpp/ripple/Peer.cpp | 10 +++++++++- src/cpp/ripple/ValidationCollection.cpp | 4 ++-- src/cpp/ripple/ValidationCollection.h | 2 +- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/src/cpp/ripple/LedgerConsensus.cpp b/src/cpp/ripple/LedgerConsensus.cpp index 971bb332ae..23744cb1ec 100644 --- a/src/cpp/ripple/LedgerConsensus.cpp +++ b/src/cpp/ripple/LedgerConsensus.cpp @@ -201,7 +201,7 @@ void LedgerConsensus::checkOurValidation() v->setTrusted(); v->sign(signingHash, mValPrivate); theApp->isNew(signingHash); - theApp->getValidations().addValidation(v); + theApp->getValidations().addValidation(v, "localMissing"); std::vector validation = v->getSigned(); ripple::TMValidation val; val.set_validation(&validation[0], validation.size()); @@ -1182,7 +1182,7 @@ void LedgerConsensus::accept(SHAMap::ref set, LoadEvent::pointer) v->sign(signingHash, mValPrivate); v->setTrusted(); theApp->isNew(signingHash); // suppress it if we receive it - theApp->getValidations().addValidation(v); + theApp->getValidations().addValidation(v, "local"); theApp->getOPs().setLastValidation(v); std::vector validation = v->getSigned(); ripple::TMValidation val; diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index 26a8da1bff..f576fea8b0 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1164,10 +1164,10 @@ std::vector return accounts; } -bool NetworkOPs::recvValidation(SerializedValidation::ref val) +bool NetworkOPs::recvValidation(SerializedValidation::ref val, const std::string& source) { - cLog(lsDEBUG) << "recvValidation " << val->getLedgerHash(); - return theApp->getValidations().addValidation(val); + cLog(lsDEBUG) << "recvValidation " << val->getLedgerHash() << " from " << source; + return theApp->getValidations().addValidation(val, source); } Json::Value NetworkOPs::getConsensusInfo() diff --git a/src/cpp/ripple/NetworkOPs.h b/src/cpp/ripple/NetworkOPs.h index 99daf9483d..00cb710c92 100644 --- a/src/cpp/ripple/NetworkOPs.h +++ b/src/cpp/ripple/NetworkOPs.h @@ -253,7 +253,7 @@ public: RippleAddress nodePublic, uint256 checkLedger, bool sigGood); SMAddNode gotTXData(const boost::shared_ptr& peer, const uint256& hash, const std::list& nodeIDs, const std::list< std::vector >& nodeData); - bool recvValidation(SerializedValidation::ref val); + bool recvValidation(SerializedValidation::ref val, const std::string& source); void takePosition(int seq, SHAMap::ref position); SHAMap::pointer getTXMap(const uint256& hash); bool hasTXSet(const boost::shared_ptr& peer, const uint256& set, ripple::TxSetStatus status); diff --git a/src/cpp/ripple/Peer.cpp b/src/cpp/ripple/Peer.cpp index 5f26291316..4a87a4dc4d 100644 --- a/src/cpp/ripple/Peer.cpp +++ b/src/cpp/ripple/Peer.cpp @@ -1056,8 +1056,16 @@ static void checkValidation(Job&, SerializedValidation::pointer val, uint256 sig return; } + std::string source; + Peer::pointer lp = peer.lock(); + if (lp) + source = lp->getDisplayName(); + else + source = "unknown"; + std::set peers; - if (theApp->getOPs().recvValidation(val) && theApp->getSuppression().swapSet(signingHash, peers, SF_RELAYED)) + if (theApp->getOPs().recvValidation(val, source) && + theApp->getSuppression().swapSet(signingHash, peers, SF_RELAYED)) { PackedMessage::pointer message = boost::make_shared(*packet, ripple::mtVALIDATION); theApp->getConnectionPool().relayMessageBut(peers, message); diff --git a/src/cpp/ripple/ValidationCollection.cpp b/src/cpp/ripple/ValidationCollection.cpp index 2827c36f52..a9ffab3e17 100644 --- a/src/cpp/ripple/ValidationCollection.cpp +++ b/src/cpp/ripple/ValidationCollection.cpp @@ -34,7 +34,7 @@ VSpointer ValidationCollection::findSet(const uint256& ledgerHash) return mValidations.fetch(ledgerHash); } -bool ValidationCollection::addValidation(SerializedValidation::ref val) +bool ValidationCollection::addValidation(SerializedValidation::ref val, const std::string& source) { RippleAddress signer = val->getSignerPublic(); bool isCurrent = false; @@ -53,7 +53,7 @@ bool ValidationCollection::addValidation(SerializedValidation::ref val) else { cLog(lsDEBUG) << "Node " << signer.humanNodePublic() << " not in UNL st=" << val->getSignTime() << - ", hash=" << val->getLedgerHash() << ", shash=" << val->getSigningHash(); + ", hash=" << val->getLedgerHash() << ", shash=" << val->getSigningHash() << " src=" << source; } uint256 hash = val->getLedgerHash(); diff --git a/src/cpp/ripple/ValidationCollection.h b/src/cpp/ripple/ValidationCollection.h index 112e9c1113..d73a546c65 100644 --- a/src/cpp/ripple/ValidationCollection.h +++ b/src/cpp/ripple/ValidationCollection.h @@ -37,7 +37,7 @@ public: ValidationCollection() : mValidations("Validations", 128, 600), mWriting(false) { mStaleValidations.reserve(512); } - bool addValidation(SerializedValidation::ref); + bool addValidation(SerializedValidation::ref, const std::string& source); ValidationSet getValidations(const uint256& ledger); void getValidationCount(const uint256& ledger, bool currentOnly, int& trusted, int& untrusted); void getValidationTypes(const uint256& ledger, int& full, int& partial); From edd79c50a940e8e8bc3f62e727ea9b9cf71d141a Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Thu, 21 Mar 2013 13:11:27 -0700 Subject: [PATCH 36/42] Add RPC json. --- src/cpp/ripple/CallRPC.cpp | 20 ++++++++++++++++++++ src/cpp/ripple/CallRPC.h | 3 ++- src/cpp/ripple/main.cpp | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/cpp/ripple/CallRPC.cpp b/src/cpp/ripple/CallRPC.cpp index 03ed58a536..662170f2a8 100644 --- a/src/cpp/ripple/CallRPC.cpp +++ b/src/cpp/ripple/CallRPC.cpp @@ -328,6 +328,25 @@ Json::Value RPCParser::parseGetCounts(const Json::Value& jvParams) return jvRequest; } +// json +Json::Value RPCParser::parseJson(const Json::Value& jvParams) +{ + Json::Reader reader; + Json::Value jvRequest; + + cLog(lsTRACE) << "RPC method: " << jvParams[0u]; + cLog(lsTRACE) << "RPC json: " << jvParams[1u]; + + if (reader.parse(jvParams[1u].asString(), jvRequest)) + { + jvRequest["method"] = jvParams[0u]; + + return jvRequest; + } + + return rpcError(rpcINVALID_PARAMS); +} + // ledger [id|index|current|closed|validated] [full] Json::Value RPCParser::parseLedger(const Json::Value& jvParams) { @@ -631,6 +650,7 @@ Json::Value RPCParser::parseCommand(std::string strMethod, Json::Value jvParams) { "connect", &RPCParser::parseConnect, 1, 2 }, { "consensus_info", &RPCParser::parseAsIs, 0, 0 }, { "get_counts", &RPCParser::parseGetCounts, 0, 1 }, + { "json", &RPCParser::parseJson, 2, 2 }, { "ledger", &RPCParser::parseLedger, 0, 2 }, { "ledger_accept", &RPCParser::parseAsIs, 0, 0 }, { "ledger_closed", &RPCParser::parseAsIs, 0, 0 }, diff --git a/src/cpp/ripple/CallRPC.h b/src/cpp/ripple/CallRPC.h index 5d9a0377fc..c87ffbe958 100644 --- a/src/cpp/ripple/CallRPC.h +++ b/src/cpp/ripple/CallRPC.h @@ -23,9 +23,10 @@ protected: #endif Json::Value parseEvented(const Json::Value& jvParams); Json::Value parseGetCounts(const Json::Value& jvParams); + Json::Value parseInternal(const Json::Value& jvParams); + Json::Value parseJson(const Json::Value& jvParams); Json::Value parseLedger(const Json::Value& jvParams); Json::Value parseLedgerId(const Json::Value& jvParams); - Json::Value parseInternal(const Json::Value& jvParams); #if ENABLE_INSECURE Json::Value parseLogin(const Json::Value& jvParams); #endif diff --git a/src/cpp/ripple/main.cpp b/src/cpp/ripple/main.cpp index 1b3a9c9df2..8a7ed48b24 100644 --- a/src/cpp/ripple/main.cpp +++ b/src/cpp/ripple/main.cpp @@ -84,6 +84,7 @@ void printHelp(const po::options_description& desc) cerr << " data_store " << endl; #endif cerr << " get_counts" << endl; + cerr << " json " << endl; cerr << " ledger [|current|closed|validated] [full]" << endl; cerr << " ledger_accept" << endl; cerr << " ledger_closed" << endl; From 4ac824419a89a23800b5bd4926ddc9ce535ea6fb Mon Sep 17 00:00:00 2001 From: Arthur Britto Date: Thu, 21 Mar 2013 16:27:24 -0700 Subject: [PATCH 37/42] Abstract valid offer testing in OfferCreate. --- src/cpp/ripple/OfferCreateTransactor.cpp | 309 +++++++++++++---------- src/cpp/ripple/OfferCreateTransactor.h | 12 + 2 files changed, 183 insertions(+), 138 deletions(-) diff --git a/src/cpp/ripple/OfferCreateTransactor.cpp b/src/cpp/ripple/OfferCreateTransactor.cpp index 686b21e832..0743b36380 100644 --- a/src/cpp/ripple/OfferCreateTransactor.cpp +++ b/src/cpp/ripple/OfferCreateTransactor.cpp @@ -7,6 +7,80 @@ SETUP_LOG(); +// Make sure an offer is still valid. If not, mark it unfunded. +bool OfferCreateTransactor::bValidOffer( + SLE::ref sleOfferDir, + const uint256& uOfferIndex, + const uint160& uOfferOwnerID, + const STAmount& saOfferPays, + const STAmount& saOfferGets, + const uint160& uTakerAccountID, + boost::unordered_set& usOfferUnfundedFound, + boost::unordered_set& usOfferUnfundedBecame, + boost::unordered_set& usAccountTouched, + STAmount& saOfferFunds) { // <-- + bool bValid; + + if (sleOfferDir->isFieldPresent(sfExpiration) && sleOfferDir->getFieldU32(sfExpiration) <= mEngine->getLedger()->getParentCloseTimeNC()) + { + // Offer is expired. Expired offers are considered unfunded. Delete it. + cLog(lsINFO) << "bValidOffer: encountered expired offer"; + + usOfferUnfundedFound.insert(uOfferIndex); + + bValid = false; + } + else if (uOfferOwnerID == uTakerAccountID) + { + // Would take own offer. Consider old offer expired. Delete it. + cLog(lsINFO) << "bValidOffer: encountered taker's own old offer"; + + usOfferUnfundedFound.insert(uOfferIndex); + + bValid = false; + } + else if (!saOfferGets.isPositive() || !saOfferPays.isPositive()) + { + // Offer has bad amounts. Consider offer expired. Delete it. + cLog(lsWARNING) << boost::str(boost::format("bValidOffer: BAD OFFER: saOfferPays=%s saOfferGets=%s") + % saOfferPays % saOfferGets); + + usOfferUnfundedFound.insert(uOfferIndex); + } + else + { + cLog(lsINFO) << "bValidOffer: saOfferPays=" << saOfferPays.getFullText(); + + saOfferFunds = mEngine->getNodes().accountFunds(uOfferOwnerID, saOfferPays); + + if (!saOfferFunds.isPositive()) + { + // Offer is unfunded, possibly due to previous balance action. + cLog(lsINFO) << "bValidOffer: offer unfunded: delete"; + + boost::unordered_set::iterator account = usAccountTouched.find(uOfferOwnerID); + if (account != usAccountTouched.end()) + { + // Previously touched account. + usOfferUnfundedBecame.insert(uOfferIndex); // Delete unfunded offer on success. + } + else + { + // Never touched source account. + usOfferUnfundedFound.insert(uOfferIndex); // Delete found unfunded offer when possible. + } + + bValid = false; + } + else + { + bValid = true; + } + } + + return bValid; +} + // Take as much as possible. Adjusts account balances. Charges fees on top to taker. // --> uBookBase: The order book to take against. // --> saTakerPays: What the taker offers (w/ issuer) @@ -28,7 +102,7 @@ TER OfferCreateTransactor::takeOffers( bool& bUnfunded) { // The book has the most elements. Take the perspective of the book. - // Book is ordered for taker: taker pays / taker gets, < is better + // Book is ordered for taker: taker pays / taker gets (smaller is better) // The order is for the other books currencys for get and pays are opposites. // We want the same ratio for the respective currencies. @@ -141,162 +215,121 @@ TER OfferCreateTransactor::takeOffers( STAmount saOfferPays = sleOffer->getFieldAmount(sfTakerGets); STAmount saOfferGets = sleOffer->getFieldAmount(sfTakerPays); - if (sleOffer->isFieldPresent(sfExpiration) && sleOffer->getFieldU32(sfExpiration) <= mEngine->getLedger()->getParentCloseTimeNC()) - { - // Offer is expired. Expired offers are considered unfunded. Delete it. - cLog(lsINFO) << "takeOffers: encountered expired offer"; + STAmount saOfferFunds; // Funds of offer owner to payout. + bool bValid; - usOfferUnfundedFound.insert(uOfferIndex); - } - else if (uOfferOwnerID == uTakerAccountID) - { - // Would take own offer. Consider old offer expired. Delete it. - cLog(lsINFO) << "takeOffers: encountered taker's own old offer"; + bValid = bValidOffer( + sleOfferDir, uOfferIndex, uOfferOwnerID, saOfferPays, saOfferGets, + uTakerAccountID, + usOfferUnfundedFound, usOfferUnfundedBecame, usAccountTouched, + saOfferFunds); - usOfferUnfundedBecame.insert(uOfferIndex); - } - else if (!saOfferGets.isPositive() || !saOfferPays.isPositive()) - { - // Offer has bad amounts. Consider offer expired. Delete it. - cLog(lsWARNING) << boost::str(boost::format("takeOffers: BAD OFFER: saOfferPays=%s saOfferGets=%s") - % saOfferPays % saOfferGets); + if (bValid) { + STAmount saSubTakerPaid; + STAmount saSubTakerGot; + STAmount saTakerIssuerFee; + STAmount saOfferIssuerFee; + STAmount saOfferRate = STAmount::setRate(uTipQuality); - usOfferUnfundedFound.insert(uOfferIndex); - } - else - { - // Get offer funds available. + cLog(lsINFO) << "takeOffers: applyOffer: saTakerPays: " << saTakerPays.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerPaid: " << saTakerPaid.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerFunds: " << saTakerFunds.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saOfferFunds: " << saOfferFunds.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saOfferPays: " << saOfferPays.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saOfferGets: " << saOfferGets.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saOfferRate: " << saOfferRate.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerPays: " << saSubTakerPays.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerGets: " << saSubTakerGets.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerPays: " << saTakerPays.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerGets: " << saTakerGets.getFullText(); - cLog(lsINFO) << "takeOffers: saOfferPays=" << saOfferPays.getFullText(); + bool bOfferDelete = STAmount::applyOffer( + lesActive.rippleTransferRate(uTakerAccountID, uOfferOwnerID, uTakerPaysAccountID), + lesActive.rippleTransferRate(uOfferOwnerID, uTakerAccountID, uTakerGetsAccountID), + saOfferRate, + saOfferFunds, + saTakerFunds, + saOfferPays, + saOfferGets, + saSubTakerPays, + saSubTakerGets, + saSubTakerPaid, + saSubTakerGot, + saTakerIssuerFee, + saOfferIssuerFee); - STAmount saOfferFunds = lesActive.accountFunds(uOfferOwnerID, saOfferPays); - SLE::pointer sleOfferAccount; // Owner of offer. + cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerPaid: " << saSubTakerPaid.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerGot: " << saSubTakerGot.getFullText(); - if (!saOfferFunds.isPositive()) // Includes zero. + // Adjust offer + + // Offer owner will pay less. Subtract what taker just got. + sleOffer->setFieldAmount(sfTakerGets, saOfferPays -= saSubTakerGot); + + // Offer owner will get less. Subtract what owner just paid. + sleOffer->setFieldAmount(sfTakerPays, saOfferGets -= saSubTakerPaid); + + mEngine->entryModify(sleOffer); + + if (bOfferDelete) { - // Offer is unfunded, possibly due to previous balance action. - cLog(lsINFO) << "takeOffers: offer unfunded: delete"; + // Offer now fully claimed or now unfunded. + cLog(lsINFO) << "takeOffers: Offer claimed: Delete."; - boost::unordered_set::iterator account = usAccountTouched.find(uOfferOwnerID); - if (account != usAccountTouched.end()) + usOfferUnfundedBecame.insert(uOfferIndex); // Delete unfunded offer on success. + + // Offer owner's account is no longer pristine. + usAccountTouched.insert(uOfferOwnerID); + } + else if (saSubTakerGot) + { + cLog(lsINFO) << "takeOffers: Offer partial claim."; + + if (!saOfferPays.isPositive() || !saOfferGets.isPositive()) { - // Previously touched account. - usOfferUnfundedBecame.insert(uOfferIndex); // Delete unfunded offer on success. - } - else - { - // Never touched source account. - usOfferUnfundedFound.insert(uOfferIndex); // Delete found unfunded offer when possible. + cLog(lsWARNING) << "takeOffers: ILLEGAL OFFER RESULT."; + bUnfunded = true; + terResult = bOpenLedger ? telFAILED_PROCESSING : tecFAILED_PROCESSING; } } else { - STAmount saSubTakerPaid; - STAmount saSubTakerGot; - STAmount saTakerIssuerFee; - STAmount saOfferIssuerFee; - STAmount saOfferRate = STAmount::setRate(uTipQuality); + // Taker got nothing, probably due to rounding. Consider taker unfunded. + cLog(lsINFO) << "takeOffers: No claim."; - cLog(lsINFO) << "takeOffers: applyOffer: saTakerPays: " << saTakerPays.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerPaid: " << saTakerPaid.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerFunds: " << saTakerFunds.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saOfferFunds: " << saOfferFunds.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saOfferPays: " << saOfferPays.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saOfferGets: " << saOfferGets.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saOfferRate: " << saOfferRate.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerPays: " << saSubTakerPays.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerGets: " << saSubTakerGets.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerPays: " << saTakerPays.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerGets: " << saTakerGets.getFullText(); + bUnfunded = true; + terResult = tesSUCCESS; // Done. + } - bool bOfferDelete = STAmount::applyOffer( - lesActive.rippleTransferRate(uTakerAccountID, uOfferOwnerID, uTakerPaysAccountID), - lesActive.rippleTransferRate(uOfferOwnerID, uTakerAccountID, uTakerGetsAccountID), - saOfferRate, - saOfferFunds, - saTakerFunds, - saOfferPays, - saOfferGets, - saSubTakerPays, - saSubTakerGets, - saSubTakerPaid, - saSubTakerGot, - saTakerIssuerFee, - saOfferIssuerFee); + assert(uTakerGetsAccountID == saSubTakerGot.getIssuer()); + assert(uTakerPaysAccountID == saSubTakerPaid.getIssuer()); - cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerPaid: " << saSubTakerPaid.getFullText(); + if (!bUnfunded) + { + // Distribute funds. The sends charge appropriate fees which are implied by offer. + + terResult = lesActive.accountSend(uOfferOwnerID, uTakerAccountID, saSubTakerGot); // Offer owner pays taker. + + if (tesSUCCESS == terResult) + terResult = lesActive.accountSend(uTakerAccountID, uOfferOwnerID, saSubTakerPaid); // Taker pays offer owner. + + // Reduce amount considered paid by taker's rate (not actual cost). + STAmount saTakerCould = saTakerPays - saTakerPaid; // Taker could pay. + if (saTakerFunds < saTakerCould) + saTakerCould = saTakerFunds; + + STAmount saTakerUsed = STAmount::multiply(saSubTakerGot, saTakerRate, saTakerPays); + + cLog(lsINFO) << "takeOffers: applyOffer: saTakerCould: " << saTakerCould.getFullText(); cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerGot: " << saSubTakerGot.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerRate: " << saTakerRate.getFullText(); + cLog(lsINFO) << "takeOffers: applyOffer: saTakerUsed: " << saTakerUsed.getFullText(); - // Adjust offer + saTakerPaid += std::min(saTakerCould, saTakerUsed); + saTakerGot += saSubTakerGot; - // Offer owner will pay less. Subtract what taker just got. - sleOffer->setFieldAmount(sfTakerGets, saOfferPays -= saSubTakerGot); - - // Offer owner will get less. Subtract what owner just paid. - sleOffer->setFieldAmount(sfTakerPays, saOfferGets -= saSubTakerPaid); - - mEngine->entryModify(sleOffer); - - if (bOfferDelete) - { - // Offer now fully claimed or now unfunded. - cLog(lsINFO) << "takeOffers: Offer claimed: Delete."; - - usOfferUnfundedBecame.insert(uOfferIndex); // Delete unfunded offer on success. - - // Offer owner's account is no longer pristine. - usAccountTouched.insert(uOfferOwnerID); - } - else if (saSubTakerGot) - { - cLog(lsINFO) << "takeOffers: Offer partial claim."; - - if (!saOfferPays.isPositive() || !saOfferGets.isPositive()) - { - cLog(lsWARNING) << "takeOffers: ILLEGAL OFFER RESULT."; - bUnfunded = true; - terResult = bOpenLedger ? telFAILED_PROCESSING : tecFAILED_PROCESSING; - } - } - else - { - // Taker got nothing, probably due to rounding. Consider taker unfunded. - cLog(lsINFO) << "takeOffers: No claim."; - - bUnfunded = true; - terResult = tesSUCCESS; // Done. - } - - assert(uTakerGetsAccountID == saSubTakerGot.getIssuer()); - assert(uTakerPaysAccountID == saSubTakerPaid.getIssuer()); - - if (!bUnfunded) - { - // Distribute funds. The sends charge appropriate fees which are implied by offer. - - terResult = lesActive.accountSend(uOfferOwnerID, uTakerAccountID, saSubTakerGot); // Offer owner pays taker. - - if (tesSUCCESS == terResult) - terResult = lesActive.accountSend(uTakerAccountID, uOfferOwnerID, saSubTakerPaid); // Taker pays offer owner. - - // Reduce amount considered paid by taker's rate (not actual cost). - STAmount saTakerCould = saTakerPays - saTakerPaid; // Taker could pay. - if (saTakerFunds < saTakerCould) - saTakerCould = saTakerFunds; - - STAmount saTakerUsed = STAmount::multiply(saSubTakerGot, saTakerRate, saTakerPays); - - cLog(lsINFO) << "takeOffers: applyOffer: saTakerCould: " << saTakerCould.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saSubTakerGot: " << saSubTakerGot.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerRate: " << saTakerRate.getFullText(); - cLog(lsINFO) << "takeOffers: applyOffer: saTakerUsed: " << saTakerUsed.getFullText(); - - saTakerPaid += std::min(saTakerCould, saTakerUsed); - saTakerGot += saSubTakerGot; - - if (tesSUCCESS == terResult) - terResult = temUNCERTAIN; - } + if (tesSUCCESS == terResult) + terResult = temUNCERTAIN; } } } diff --git a/src/cpp/ripple/OfferCreateTransactor.h b/src/cpp/ripple/OfferCreateTransactor.h index 0947c656fd..dfe8696bca 100644 --- a/src/cpp/ripple/OfferCreateTransactor.h +++ b/src/cpp/ripple/OfferCreateTransactor.h @@ -6,6 +6,18 @@ class OfferCreateTransactor : public Transactor { protected: + bool bValidOffer( + SLE::ref sleOfferDir, + const uint256& uOffer, + const uint160& uOfferOwnerID, + const STAmount& saOfferPays, + const STAmount& saOfferGets, + const uint160& uTakerAccountID, + boost::unordered_set& usOfferUnfundedFound, + boost::unordered_set& usOfferUnfundedBecame, + boost::unordered_set& usAccountTouched, + STAmount& saOfferFunds); + TER takeOffers( const bool bOpenLedger, const bool bPassive, From 2c128d2ae07ede0eeab952c8585433988df0fe52 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Mar 2013 08:09:37 -0700 Subject: [PATCH 38/42] Tune some other DB cache sizes. --- src/cpp/ripple/Application.cpp | 4 ++++ src/cpp/ripple/Config.cpp | 4 +++- src/cpp/ripple/Config.h | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/cpp/ripple/Application.cpp b/src/cpp/ripple/Application.cpp index 7b5dd2191a..f62cb0632d 100644 --- a/src/cpp/ripple/Application.cpp +++ b/src/cpp/ripple/Application.cpp @@ -184,6 +184,10 @@ void Application::setup() theApp->getHashNodeDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % (theConfig.getSize(siHashNodeDBCache) * 1024))); + theApp->getLedgerDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % + (theConfig.getSize(siTxnDBCache) * 1024))); + theApp->getTxnDB()->getDB()->executeSQL(boost::str(boost::format("PRAGMA cache_size=-%d;") % + (theConfig.getSize(siLgrDBCache) * 1024))); // // Allow peer connections. diff --git a/src/cpp/ripple/Config.cpp b/src/cpp/ripple/Config.cpp index 8afef28738..096683d3f7 100644 --- a/src/cpp/ripple/Config.cpp +++ b/src/cpp/ripple/Config.cpp @@ -498,7 +498,9 @@ int Config::getSize(SizedItemName item) { siLineCacheSize, { 8192, 32768, 131072, 1048576, 0 } }, { siLineCacheAge, { 500, 600, 1800, 3600, 7200 } }, { siHashNodeDBCache, { 24, 48, 64, 128, 256 } }, - }; + { siTxnDBCache, { 4, 8, 32, 64, 128 } }, + { siLgrDBCache, { 4, 8, 32, 64, 128 } } + }; for (int i = 0; i < (sizeof(sizeTable) / sizeof(SizedItem)); ++i) { diff --git a/src/cpp/ripple/Config.h b/src/cpp/ripple/Config.h index 0ceceba598..f6e65856f5 100644 --- a/src/cpp/ripple/Config.h +++ b/src/cpp/ripple/Config.h @@ -65,7 +65,9 @@ enum SizedItemName siLedgerFetch, siLineCacheSize, siLineCacheAge, - siHashNodeDBCache + siHashNodeDBCache, + siTxnDBCache, + siLgrDBCache }; struct SizedItem From 433a33626941c94dde364f142fdba093d3c2cdbd Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Mar 2013 08:09:48 -0700 Subject: [PATCH 39/42] Change some fields in the way estimated close times are presented in JSON. --- src/cpp/ripple/Ledger.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/cpp/ripple/Ledger.cpp b/src/cpp/ripple/Ledger.cpp index fd34ddd4a0..737bda4803 100644 --- a/src/cpp/ripple/Ledger.cpp +++ b/src/cpp/ripple/Ledger.cpp @@ -842,14 +842,11 @@ Json::Value Ledger::getJson(int options) if (mCloseTime != 0) { + ledger["close_time"] = mCloseTime; + ledger["close_time_human"] = boost::posix_time::to_simple_string(ptFromSeconds(mCloseTime)); + ledger["close_time_resolution"] = mCloseResolution; if ((mCloseFlags & sLCF_NoConsensusTime) != 0) - ledger["close_time_estimate"] = boost::posix_time::to_simple_string(ptFromSeconds(mCloseTime)); - else - { - ledger["close_time"] = mCloseTime; - ledger["close_time_human"] = boost::posix_time::to_simple_string(ptFromSeconds(mCloseTime)); - ledger["close_time_resolution"] = mCloseResolution; - } + ledger["close_time_estimated"] = true; } } else From 84d6210f4f143ef872b42431fd4745a9cadab75a Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Mar 2013 10:28:40 -0700 Subject: [PATCH 40/42] Don't spin up acquire engine if we still need the network ledger. --- src/cpp/ripple/LedgerMaster.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/cpp/ripple/LedgerMaster.cpp b/src/cpp/ripple/LedgerMaster.cpp index 50e96b3470..4d4920f746 100644 --- a/src/cpp/ripple/LedgerMaster.cpp +++ b/src/cpp/ripple/LedgerMaster.cpp @@ -289,6 +289,9 @@ bool LedgerMaster::shouldAcquire(uint32 currentLedger, uint32 ledgerHistory, uin void LedgerMaster::resumeAcquiring() { + if (theApp->getOPs().isNeedNetworkLedger()) + return; + boost::recursive_mutex::scoped_lock ml(mLock); if (mMissingLedger && mMissingLedger->isDone()) @@ -355,6 +358,9 @@ void LedgerMaster::setFullLedger(Ledger::pointer ledger) cLog(lsDEBUG) << "Ledger " << ledger->getLedgerSeq() << " accepted :" << ledger->getHash(); assert(ledger->peekAccountStateMap()->getHash().isNonZero()); + if (theApp->getOPs().isNeedNetworkLedger()) + return; + boost::recursive_mutex::scoped_lock ml(mLock); mCompleteLedgers.setValue(ledger->getLedgerSeq()); From f6172d3d458087633af76117c23e4e04cd874437 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Mar 2013 13:34:52 -0700 Subject: [PATCH 41/42] subLedger should report the validated ledger not the closed one. --- src/cpp/ripple/NetworkOPs.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index f576fea8b0..ce08b59dc2 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1665,18 +1665,20 @@ void NetworkOPs::unsubAccountChanges(InfoSub* isrListener) // <-- bool: true=added, false=already there bool NetworkOPs::subLedger(InfoSub::ref isrListener, Json::Value& jvResult) { - Ledger::pointer lpClosed = getClosedLedger(); + Ledger::pointer lpClosed = getValidatedLedger(); + if (lpClosed) + { + jvResult["ledger_index"] = lpClosed->getLedgerSeq(); + jvResult["ledger_hash"] = lpClosed->getHash().ToString(); + jvResult["ledger_time"] = Json::Value::UInt(lpClosed->getCloseTimeNC()); - jvResult["ledger_index"] = lpClosed->getLedgerSeq(); - jvResult["ledger_hash"] = lpClosed->getHash().ToString(); - jvResult["ledger_time"] = Json::Value::UInt(lpClosed->getCloseTimeNC()); + jvResult["fee_ref"] = Json::UInt(lpClosed->getReferenceFeeUnits()); + jvResult["fee_base"] = Json::UInt(lpClosed->getBaseFee()); + jvResult["reserve_base"] = Json::UInt(lpClosed->getReserve(0)); + jvResult["reserve_inc"] = Json::UInt(lpClosed->getReserveInc()); + } - jvResult["fee_ref"] = Json::UInt(lpClosed->getReferenceFeeUnits()); - jvResult["fee_base"] = Json::UInt(lpClosed->getBaseFee()); - jvResult["reserve_base"] = Json::UInt(lpClosed->getReserve(0)); - jvResult["reserve_inc"] = Json::UInt(lpClosed->getReserveInc()); - - if ((mMode == omFULL) || (mMode == omTRACKING)) + if ((mMode == omFULL) || (mMode == omTRACKING) && !isNeedNetworkLedger()) jvResult["validated_ledgers"] = theApp->getLedgerMaster().getCompleteLedgers(); boost::recursive_mutex::scoped_lock sl(mMonitorLock); From 7e5d4b80d964d14bba6579e9810e053702e7a3b7 Mon Sep 17 00:00:00 2001 From: JoelKatz Date: Fri, 22 Mar 2013 13:37:02 -0700 Subject: [PATCH 42/42] Parenthesis. --- src/cpp/ripple/NetworkOPs.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/ripple/NetworkOPs.cpp b/src/cpp/ripple/NetworkOPs.cpp index ce08b59dc2..7b46154400 100644 --- a/src/cpp/ripple/NetworkOPs.cpp +++ b/src/cpp/ripple/NetworkOPs.cpp @@ -1678,7 +1678,7 @@ bool NetworkOPs::subLedger(InfoSub::ref isrListener, Json::Value& jvResult) jvResult["reserve_inc"] = Json::UInt(lpClosed->getReserveInc()); } - if ((mMode == omFULL) || (mMode == omTRACKING) && !isNeedNetworkLedger()) + if (((mMode == omFULL) || (mMode == omTRACKING)) && !isNeedNetworkLedger()) jvResult["validated_ledgers"] = theApp->getLedgerMaster().getCompleteLedgers(); boost::recursive_mutex::scoped_lock sl(mMonitorLock);