Merge branch 'master' of github.com:jedmccaleb/NewCoin

This commit is contained in:
Arthur Britto
2013-01-09 16:22:10 -08:00
10 changed files with 184 additions and 144 deletions

View File

@@ -12,7 +12,8 @@ SETUP_LOG();
DECLARE_INSTANCE(HashedObject);
HashedObjectStore::HashedObjectStore(int cacheSize, int cacheAge) :
mCache("HashedObjectStore", cacheSize, cacheAge), mWriteGeneration(0), mWritePending(false)
mCache("HashedObjectStore", cacheSize, cacheAge), mNegativeCache("HashedObjectNegativeCache", 0, 120),
mWriteGeneration(0), mWritePending(false)
{
mWriteSet.reserve(128);
}
@@ -33,7 +34,6 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
}
assert(hash == Serializer::getSHA512Half(data));
mNegativeCache.del(hash);
HashedObject::pointer object = boost::make_shared<HashedObject>(type, index, data, hash);
if (!mCache.canonicalize(hash, object))
{
@@ -48,6 +48,7 @@ bool HashedObjectStore::store(HashedObjectType type, uint32 index,
}
// else
// cLog(lsTRACE) << "HOS: already had " << hash;
mNegativeCache.del(hash);
return true;
}
@@ -152,6 +153,7 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
db->getStr("ObjType", type);
if (type.size() == 0)
{
assert(false);
mNegativeCache.add(hash);
return HashedObject::pointer();
}
@@ -173,6 +175,7 @@ HashedObject::pointer HashedObjectStore::retrieve(const uint256& hash)
case 'A': htype = hotACCOUNT_NODE; break;
case 'N': htype = hotTRANSACTION_NODE; break;
default:
assert(false);
cLog(lsERROR) << "Invalid hashed object";
mNegativeCache.add(hash);
return HashedObject::pointer();

View File

@@ -1,6 +1,8 @@
#ifndef KEY_CACHE__H
#define KEY_CACHE__H
#include <string>
#include <boost/unordered_map.hpp>
#include <boost/thread/mutex.hpp>
@@ -12,40 +14,58 @@ public:
typedef typename map_type::iterator map_iterator;
protected:
boost::mutex mNCLock;
map_type mCache;
int mTargetSize, mTargetAge;
const std::string mName;
boost::mutex mNCLock;
map_type mCache;
int mTargetSize, mTargetAge;
uint64_t mHits, mMisses;
public:
KeyCache(int size = 0, int age = 120) : mTargetSize(size), mTargetAge(age), mHits(0), mMisses(0)
KeyCache(const std::string& name, int size = 0, int age = 120) : mName(name), mTargetSize(size), mTargetAge(age)
{
assert((mTargetSize >= 0) && (mTargetAge > 2));
}
void getStats(int& size, uint64_t& hits, uint64_t& misses)
void getSize()
{
boost::mutex::scoped_lock sl(mNCLock);
size = mCache.size();
hits = mHits;
misses = mMisses;
return mCache.size();
}
bool isPresent(const key_type& key)
void getTargetSize()
{
boost::mutex::scoped_lock sl(mNCLock);
return mTargetSize;
}
void getTargetAge()
{
boost::mutex::scoped_lock sl(mNCLock);
return mTargetAge;
}
void setTargets(int size, int age)
{
boost::mutex::scoped_lock sl(mNCLock);
mTargetSize = size;
mTargetAge = age;
assert((mTargetSize >= 0) && (mTargetAge > 2));
}
const std::string& getName()
{
return mName;
}
bool isPresent(const key_type& key, bool refresh = true)
{ // Check if an entry is cached, refresh it if so
boost::mutex::scoped_lock sl(mNCLock);
map_iterator it = mCache.find(key);
if (it == mCache.end())
{
++mMisses;
return false;
}
it->second = time(NULL);
++mHits;
if (refresh)
it->second = time(NULL);
return true;
}

View File

@@ -525,7 +525,7 @@ Ledger::pointer Ledger::getSQL(const std::string& sql)
assert(false);
return Ledger::pointer();
}
Log(lsTRACE) << "Loaded ledger: " << ledgerHash;
cLog(lsTRACE) << "Loaded ledger: " << ledgerHash;
return ret;
}
@@ -538,28 +538,11 @@ Ledger::pointer Ledger::loadByIndex(uint32 ledgerIndex)
}
Ledger::pointer Ledger::loadByHash(const uint256& ledgerHash)
{ // This is a low-level function with no caching
{ // This is a low-level function with no caching and only gets accepted ledgers
std::string sql="SELECT * from Ledgers WHERE LedgerHash='";
sql.append(ledgerHash.GetHex());
sql.append("';");
Ledger::pointer ret = getSQL(sql);
if (ret)
return ret;
HashedObject::pointer node = theApp->getHashedObjectStore().retrieve(ledgerHash);
if (!node)
return Ledger::pointer();
try
{
Ledger::pointer ledger = boost::make_shared<Ledger>(strCopy(node->getData()), true);
if (ledger->getHash() == ledgerHash)
return ledger;
}
catch (...)
{
cLog(lsDEBUG) << "Exception trying to load ledger by hash: " << ledgerHash;
return Ledger::pointer();
}
return Ledger::pointer();
return getSQL(sql);
}
Ledger::pointer Ledger::getLastFullLedger()

View File

@@ -74,8 +74,9 @@ void PeerSet::TimerEntry(boost::weak_ptr<PeerSet> wptr, const boost::system::err
ptr->invokeOnTimer();
}
LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT),
mHaveBase(false), mHaveState(false), mHaveTransactions(false), mAborted(false), mSignaled(false), mAccept(false)
LedgerAcquire::LedgerAcquire(const uint256& hash) : PeerSet(hash, LEDGER_ACQUIRE_TIMEOUT),
mHaveBase(false), mHaveState(false), mHaveTransactions(false), mAborted(false), mSignaled(false), mAccept(false),
mByHash(true)
{
#ifdef LA_DEBUG
cLog(lsTRACE) << "Acquiring ledger " << mHash;
@@ -90,7 +91,12 @@ bool LedgerAcquire::tryLocal()
return false;
mLedger = boost::make_shared<Ledger>(strCopy(node->getData()), true);
assert(mLedger->getHash() == mHash);
if (mLedger->getHash() != mHash)
{ // We know for a fact the ledger can never be acquired
cLog(lsWARNING) << mHash << " cannot be a ledger";
mFailed = true;
return true;
}
mHaveBase = true;
if (!mLedger->getTransHash())
@@ -126,6 +132,7 @@ void LedgerAcquire::onTimer(bool progress)
{
if (getTimeouts() > 6)
{
cLog(lsWARNING) << "Six timeouts for ledger " << mHash;
setFailed();
done();
return;
@@ -133,11 +140,14 @@ void LedgerAcquire::onTimer(bool progress)
if (!progress)
{
cLog(lsDEBUG) << "No progress for ledger " << mHash;
if (!getPeerCount())
addPeers();
else
trigger(Peer::pointer());
}
else
mByHash = true;
}
void LedgerAcquire::addPeers()
@@ -184,13 +194,13 @@ void LedgerAcquire::done()
mOnComplete.clear();
mLock.unlock();
if (isComplete() && mLedger)
if (isComplete() && !isFailed() && mLedger)
{
if (mAccept)
mLedger->setAccepted();
theApp->getLedgerMaster().storeLedger(mLedger);
}
else if (isFailed())
else
theApp->getMasterLedgerAcquire().logFailure(mHash);
for (unsigned int i = 0; i < triggers.size(); ++i)
@@ -226,7 +236,13 @@ void LedgerAcquire::trigger(Peer::ref peer)
}
if (!mHaveBase)
{
tryLocal();
if (mFailed)
{
cLog(lsWARNING) << " failed local for " << mHash;
}
}
ripple::TMGetLedger tmGL;
tmGL.set_ledgerhash(mHash.begin(), mHash.size());
@@ -234,7 +250,7 @@ void LedgerAcquire::trigger(Peer::ref peer)
{
tmGL.set_querytype(ripple::qtINDIRECT);
if (!isProgress())
if (!isProgress() && !mFailed && mByHash)
{
std::vector<neededHash_t> need = getNeededHashes();
if (!need.empty())
@@ -260,9 +276,6 @@ void LedgerAcquire::trigger(Peer::ref peer)
}
}
PackedMessage::pointer packet = boost::make_shared<PackedMessage>(tmBH, ripple::mtGET_OBJECTS);
if (peer)
peer->sendPacket(packet);
else
{
boost::recursive_mutex::scoped_lock sl(mLock);
for (boost::unordered_map<uint64, int>::iterator it = mPeers.begin(), end = mPeers.end();
@@ -270,15 +283,24 @@ void LedgerAcquire::trigger(Peer::ref peer)
{
Peer::pointer iPeer = theApp->getConnectionPool().getPeerById(it->first);
if (iPeer)
{
mByHash = false;
iPeer->sendPacket(packet);
}
}
}
}
else
{
cLog(lsINFO) << "getNeededHashes says acquire is complete";
mHaveBase = true;
mHaveTransactions = true;
mHaveState = true;
}
}
}
if (!mHaveBase)
if (!mHaveBase && !mFailed)
{
tmGL.set_itype(ripple::liBASE);
cLog(lsTRACE) << "Sending base request to " << (peer ? "selected peer" : "all peers");
@@ -290,7 +312,7 @@ void LedgerAcquire::trigger(Peer::ref peer)
if (mLedger)
tmGL.set_ledgerseq(mLedger->getLedgerSeq());
if (mHaveBase && !mHaveTransactions)
if (mHaveBase && !mHaveTransactions && !mFailed)
{
assert(mLedger);
if (mLedger->peekTransactionMap()->getHash().isZero())
@@ -331,7 +353,7 @@ void LedgerAcquire::trigger(Peer::ref peer)
}
}
if (mHaveBase && !mHaveState)
if (mHaveBase && !mHaveState && !mFailed)
{
assert(mLedger);
if (mLedger->peekAccountStateMap()->getHash().isZero())
@@ -703,57 +725,10 @@ SMAddNode LedgerAcquireMaster::gotLedgerData(ripple::TMLedgerData& packet, Peer:
return SMAddNode::invalid();
}
void LedgerAcquireMaster::logFailure(const uint256& hash)
{
time_t now = time(NULL);
boost::mutex::scoped_lock sl(mLock);
std::map<uint256, time_t>::iterator it = mRecentFailures.begin();
while (it != mRecentFailures.end())
{
if (it->first == hash)
{
it->second = now;
return;
}
if (it->second > now)
{ // time jump or discontinuity
it->second = now;
++it;
}
else if ((it->second + 180) < now)
mRecentFailures.erase(it++);
else
++it;
}
mRecentFailures[hash] = now;
}
bool LedgerAcquireMaster::isFailure(const uint256& hash)
{
time_t now = time(NULL);
boost::mutex::scoped_lock sl(mLock);
std::map<uint256, time_t>::iterator it = mRecentFailures.find(hash);
if (it == mRecentFailures.end())
return false;
if (it->second > now)
{
it->second = now;
return true;
}
if ((it->second + 180) < now)
{
mRecentFailures.erase(it);
return false;
}
return true;
}
void LedgerAcquireMaster::sweep()
{
mRecentFailures.sweep();
time_t now = time(NULL);
boost::mutex::scoped_lock sl(mLock);

View File

@@ -18,6 +18,11 @@
#include "InstanceCounter.h"
#include "ripple.pb.h"
// How long before we try again to acquire the same ledger
#ifndef LEDGER_REACQUIRE_INTERVAL
#define LEDGER_REACQUIRE_INTERVAL 180
#endif
DEFINE_INSTANCE(LedgerAcquire);
class PeerSet
@@ -78,7 +83,7 @@ public:
protected:
Ledger::pointer mLedger;
bool mHaveBase, mHaveState, mHaveTransactions, mAborted, mSignaled, mAccept;
bool mHaveBase, mHaveState, mHaveTransactions, mAborted, mSignaled, mAccept, mByHash;
std::vector< boost::function<void (LedgerAcquire::pointer)> > mOnComplete;
@@ -123,10 +128,10 @@ class LedgerAcquireMaster
protected:
boost::mutex mLock;
std::map<uint256, LedgerAcquire::pointer> mLedgers;
std::map<uint256, time_t> mRecentFailures;
KeyCache<uint256> mRecentFailures;
public:
LedgerAcquireMaster() { ; }
LedgerAcquireMaster() : mRecentFailures("LedgerAcquireRecentFailures", 0, LEDGER_REACQUIRE_INTERVAL) { ; }
LedgerAcquire::pointer findCreate(const uint256& hash);
LedgerAcquire::pointer find(const uint256& hash);
@@ -134,8 +139,8 @@ public:
void dropLedger(const uint256& ledgerHash);
SMAddNode gotLedgerData(ripple::TMLedgerData& packet, Peer::ref);
void logFailure(const uint256&);
bool isFailure(const uint256&);
void logFailure(const uint256& h) { mRecentFailures.add(h); }
bool isFailure(const uint256& h) { return mRecentFailures.isPresent(h, false); }
void sweep();
};

View File

@@ -1190,15 +1190,24 @@ uint32 LedgerConsensus::roundCloseTime(uint32 closeTime)
void LedgerConsensus::accept(SHAMap::ref set, LoadEvent::pointer)
{
if (set->getHash().isNonZero())
if (set->getHash().isNonZero()) // put our set where others can get it later
theApp->getOPs().takePosition(mPreviousLedger->getLedgerSeq(), set);
boost::recursive_mutex::scoped_lock masterLock(theApp->getMasterLock());
assert(set->getHash() == mOurPosition->getCurrentHash());
uint32 closeTime = roundCloseTime(mOurPosition->getCloseTime());
bool closeTimeCorrect = true;
if (closeTime == 0)
{ // we agreed to disagree
closeTimeCorrect = false;
closeTime = mPreviousLedger->getCloseTimeNC() + 1;
}
cLog(lsDEBUG) << "Computing new LCL based on network consensus";
cLog(lsDEBUG) << "Report: Prop=" << (mProposing ? "yes" : "no") << " val=" << (mValidating ? "yes" : "no") <<
" corLCL=" << (mHaveCorrectLCL ? "yes" : "no") << " fail="<< (mConsensusFail ? "yes" : "no");
cLog(lsDEBUG) << "Report: Prev = " << mPrevLedgerHash << ":" << mPreviousLedger->getLedgerSeq();
cLog(lsDEBUG) << "Report: TxSt = " << set->getHash() << ", close " << closeTime << (closeTimeCorrect ? "" : "X");
CanonicalTXSet failedTransactions(set->getHash());
@@ -1212,7 +1221,6 @@ void LedgerConsensus::accept(SHAMap::ref set, LoadEvent::pointer)
boost::shared_ptr<SHAMap::SHADirtyMap> acctNodes = newLCL->peekAccountStateMap()->disarmDirty();
boost::shared_ptr<SHAMap::SHADirtyMap> txnNodes = newLCL->peekTransactionMap()->disarmDirty();
// write out dirty nodes (temporarily done here) Most come before setAccepted
int fc;
while ((fc = SHAMap::flushDirty(*acctNodes, 256, hotACCOUNT_NODE, newLCL->getLedgerSeq())) > 0)
@@ -1220,17 +1228,6 @@ void LedgerConsensus::accept(SHAMap::ref set, LoadEvent::pointer)
while ((fc = SHAMap::flushDirty(*txnNodes, 256, hotTRANSACTION_NODE, newLCL->getLedgerSeq())) > 0)
{ cLog(lsTRACE) << "Flushed " << fc << " dirty transaction nodes"; }
bool closeTimeCorrect = true;
if (closeTime == 0)
{ // we agreed to disagree
closeTimeCorrect = false;
closeTime = mPreviousLedger->getCloseTimeNC() + 1;
}
cLog(lsDEBUG) << "Report: Prop=" << (mProposing ? "yes" : "no") << " val=" << (mValidating ? "yes" : "no") <<
" corLCL=" << (mHaveCorrectLCL ? "yes" : "no") << " fail="<< (mConsensusFail ? "yes" : "no");
cLog(lsDEBUG) << "Report: Prev = " << mPrevLedgerHash << ":" << mPreviousLedger->getLedgerSeq();
cLog(lsDEBUG) << "Report: TxSt = " << set->getHash() << ", close " << closeTime << (closeTimeCorrect ? "" : "X");
cLog(lsDEBUG) << "Report: NewL = " << newLCL->getHash() << ":" << newLCL->getLedgerSeq();
newLCL->setAccepted(closeTime, mCloseResolution, closeTimeCorrect);

View File

@@ -142,7 +142,7 @@ void LedgerMaster::asyncAccept(Ledger::pointer ledger)
if ((ledger->getLedgerSeq() == 0) || mCompleteLedgers.hasValue(ledger->getLedgerSeq() - 1))
break;
}
Ledger::pointer prevLedger = Ledger::loadByIndex(ledger->getLedgerSeq() - 1);
Ledger::pointer prevLedger = mLedgerHistory.getLedgerBySeq(ledger->getLedgerSeq() - 1);
if (!prevLedger || (prevLedger->getHash() != ledger->getParentHash()))
break;
ledger = prevLedger;
@@ -152,7 +152,7 @@ void LedgerMaster::asyncAccept(Ledger::pointer ledger)
bool LedgerMaster::acquireMissingLedger(const uint256& ledgerHash, uint32 ledgerSeq)
{ // return: false = already gave up recently
Ledger::pointer ledger = Ledger::loadByIndex(ledgerSeq);
Ledger::pointer ledger = mLedgerHistory.getLedgerBySeq(ledgerSeq);
if (ledger && (ledger->getHash() == ledgerHash))
{
cLog(lsDEBUG) << "Ledger hash found in database";
@@ -196,7 +196,7 @@ void LedgerMaster::missingAcquireComplete(LedgerAcquire::pointer acq)
mMissingLedger.reset();
mMissingSeq = 0;
if (!acq->isFailed())
if (acq->isComplete())
{
setFullLedger(acq->getLedger());
acq->getLedger()->pendSave(false);
@@ -383,6 +383,8 @@ void LedgerMaster::checkAccept(const uint256& hash, uint32 seq)
if (theApp->getValidations().getTrustedValidationCount(hash) < minVal) // nothing we can do
return;
cLog(lsINFO) << "Advancing accepted ledger to " << seq << " with >= " << minVal << " validations";
mLastValidateHash = hash;
mLastValidateSeq = seq;
@@ -442,19 +444,26 @@ void LedgerMaster::tryPublish()
}
else
{
LedgerAcquire::pointer acq = theApp->getMasterLedgerAcquire().findCreate(hash);
if (!acq->isDone())
if (theApp->getMasterLedgerAcquire().isFailure(hash))
{
acq->setAccept();
break;
}
else if (acq->isComplete() && !acq->isFailed())
{
mPubLedger = acq->getLedger();
mPubLedgers.push_back(mPubLedger);
cLog(lsFATAL) << "Unable to acquire a recent validated ledger";
}
else
cLog(lsWARNING) << "Failed to acquire a published ledger";
{
LedgerAcquire::pointer acq = theApp->getMasterLedgerAcquire().findCreate(hash);
if (!acq->isDone())
{
acq->setAccept();
break;
}
else if (acq->isComplete() && !acq->isFailed())
{
mPubLedger = acq->getLedger();
mPubLedgers.push_back(mPubLedger);
}
else
cLog(lsWARNING) << "Failed to acquire a published ledger";
}
}
}
}

View File

@@ -1089,7 +1089,7 @@ void Peer::recvGetObjectByHash(ripple::TMGetObjectByHash& packet)
{ // this is a query
ripple::TMGetObjectByHash reply;
reply.clear_query();
reply.set_query(false);
if (packet.has_seq())
reply.set_seq(packet.seq());
reply.set_type(packet.type());
@@ -1117,7 +1117,7 @@ void Peer::recvGetObjectByHash(ripple::TMGetObjectByHash& packet)
}
}
}
cLog(lsTRACE) << "GetObjByHash query: had " << reply.objects_size() << " of " << packet.objects_size()
cLog(lsTRACE) << "GetObjByHash had " << reply.objects_size() << " of " << packet.objects_size()
<< " for " << getIP();
sendPacket(boost::make_shared<PackedMessage>(packet, ripple::mtGET_OBJECTS));
}

View File

@@ -32,12 +32,12 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
return;
}
std::stack<SHAMapTreeNode::pointer> stack;
stack.push(root);
std::stack<SHAMapTreeNode*> stack;
stack.push(root.get());
while (!stack.empty())
{
SHAMapTreeNode::pointer node = stack.top();
SHAMapTreeNode* node = stack.top();
stack.pop();
int base = rand() % 256;
@@ -49,10 +49,10 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
{
SHAMapNode childID = node->getChildNodeID(branch);
const uint256& childHash = node->getChildHash(branch);
SHAMapTreeNode::pointer d;
SHAMapTreeNode* d = NULL;
try
{
d = getNode(childID, childHash, false);
d = getNodePointer(childID, childHash);
}
catch (SHAMapMissingNode&)
{ // node is not in the map
@@ -61,9 +61,11 @@ void SHAMap::getMissingNodes(std::vector<SHAMapNode>& nodeIDs, std::vector<uint2
std::vector<unsigned char> nodeData;
if (filter->haveNode(childID, childHash, nodeData))
{
d = boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq, snfPREFIX, childHash);
SHAMapTreeNode::pointer ptr =
boost::make_shared<SHAMapTreeNode>(childID, nodeData, mSeq, snfPREFIX, childHash);
cLog(lsTRACE) << "Got sync node from cache: " << *d;
mTNByID[*d] = d;
mTNByID[*ptr] = ptr;
d = ptr.get();
}
}
}
@@ -113,10 +115,9 @@ std::vector<uint256> SHAMap::getNeededHashes(int max)
{
SHAMapNode childID = node->getChildNodeID(branch);
const uint256& childHash = node->getChildHash(branch);
SHAMapTreeNode* d;
try
{
d = getNodePointer(childID, childHash);
SHAMapTreeNode* d = getNodePointer(childID, childHash);
assert(d);
if (d->isInner() && !d->isFullBelow())
stack.push(d);

View File

@@ -72,6 +72,53 @@ buster.testCase("Offer tests", {
});
},
"offer create then crossing offer, no trust lines" :
function (done) {
var self = this;
async.waterfall([
function (callback) {
self.remote.transaction()
.offer_create("root", "500/BTC/root", "100/USD/root")
.on('proposed', function (m) {
// console.log("PROPOSED: offer_create: %s", JSON.stringify(m));
callback(m.result !== 'tesSUCCESS');
})
.on('final', function (m) {
// console.log("FINAL: offer_create: %s", JSON.stringify(m));
buster.assert.equals('tesSUCCESS', m.metadata.TransactionResult);
callback();
})
.submit();
},
function (callback) {
self.remote.transaction()
.offer_create("root", "100/USD/root", "500/BTC/root")
.on('proposed', function (m) {
// console.log("PROPOSED: offer_create: %s", JSON.stringify(m));
callback(m.result !== 'tesSUCCESS');
})
.on('final', function (m) {
// console.log("FINAL: offer_create: %s", JSON.stringify(m));
buster.assert.equals('tesSUCCESS', m.metadata.TransactionResult);
callback();
})
.submit();
}
], function (error) {
// console.log("result: error=%s", error);
buster.refute(error);
if (error) done();
});
},
"offer_create then ledger_accept then offer_cancel then ledger_accept." :
function (done) {
var self = this;