Merge branch 'master' into develop

Conflicts:
	Builds/VisualStudio2012/RippleD.vcxproj
	modules/ripple_app/ripple_app.cpp
	modules/ripple_websocket/ripple_websocket.cpp
	src/cpp/ripple/NetworkOPs.cpp
	src/cpp/ripple/ripple_Application.cpp
	src/cpp/ripple/ripple_Peer.cpp

Builds/VisualStudio2012/RippleD.vcxproj.filters

@@ -822,6 +822,18 @@
     <ClCompile Include="..\..\modules\ripple_app\ripple_app_pt8.cpp">
       <Filter>[1] Ripple\ripple_app</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\ripple_app_pt5.cpp">
+      <Filter>1. Modules\ripple_app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\ripple_app_pt6.cpp">
+      <Filter>1. Modules\ripple_app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\ripple_app_pt7.cpp">
+      <Filter>1. Modules\ripple_app</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\modules\ripple_app\ripple_app_pt8.cpp">
+      <Filter>1. Modules\ripple_app</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\Subtrees\sqlite\sqlite3.h">

modules/ripple_data/protocol/ripple_SerializedObject.cpp

@@ -379,7 +379,10 @@ bool STObject::isEquivalent (const SerializedType& t) const
     const STObject* v = dynamic_cast<const STObject*> (&t);
 
     if (!v)
+    {
+        WriteLog (lsDEBUG, STObject) << "notEquiv " << getFullText() << " not object";
         return false;
+    }
 
     boost::ptr_vector<SerializedType>::const_iterator it1 = mData.begin (), end1 = mData.end ();
     boost::ptr_vector<SerializedType>::const_iterator it2 = v->mData.begin (), end2 = v->mData.end ();
@@ -387,7 +390,19 @@ bool STObject::isEquivalent (const SerializedType& t) const
     while ((it1 != end1) && (it2 != end2))
     {
         if ((it1->getSType () != it2->getSType ()) || !it1->isEquivalent (*it2))
+        {
+            if (it1->getSType () != it2->getSType ())
+            {
+                WriteLog (lsDEBUG, STObject) << "notEquiv type " << it1->getFullText() << " != "
+                    <<  it2->getFullText();
+            }
+            else
+            {
+                WriteLog (lsDEBUG, STObject) << "notEquiv " << it1->getFullText() << " != "
+                    <<  it2->getFullText();
+            }
             return false;
+        }
 
         ++it1;
         ++it2;
@@ -1164,7 +1179,10 @@ bool STArray::isEquivalent (const SerializedType& t) const
     const STArray* v = dynamic_cast<const STArray*> (&t);
 
     if (!v)
+    {
+        WriteLog (lsDEBUG, STObject) << "notEquiv " << getFullText() << " not array";
         return false;
+    }
 
     return value == v->value;
 }

modules/ripple_data/protocol/ripple_SerializedTypes.cpp

@@ -22,6 +22,14 @@ SerializedType& SerializedType::operator= (const SerializedType& t)
     return *this;
 }
 
+bool SerializedType::isEquivalent (const SerializedType& t) const
+{
+    assert (getSType () == STI_NOTPRESENT);
+    if (t.getSType () == STI_NOTPRESENT)
+        return true;
+    WriteLog (lsDEBUG, SerializedType) << "notEquiv " << getFullText() << " not STI_NOTPRESENT";
+    return false;
+}
 
 void STPathSet::printDebug ()
 {

modules/ripple_data/protocol/ripple_SerializedTypes.h

@@ -113,11 +113,7 @@ public:
         ;
     }
 
-    virtual bool isEquivalent (const SerializedType& t) const
+    virtual bool isEquivalent (const SerializedType& t) const;
-    {
-        assert (getSType () == STI_NOTPRESENT);
-        return t.getSType () == STI_NOTPRESENT;
-    }
 
     void addFieldID (Serializer& s) const
     {

src/cpp/ripple/LedgerMaster.cpp

@@ -25,6 +25,7 @@ Ledger::ref LedgerMaster::getCurrentSnapshot ()
 
 int LedgerMaster::getValidatedLedgerAge ()
 {
+    boost::recursive_mutex::scoped_lock ml (mLock);
     if (!mValidLedger)
     {
         WriteLog (lsDEBUG, LedgerMaster) << "No validated ledger";
@@ -39,6 +40,27 @@ int LedgerMaster::getValidatedLedgerAge ()
     return static_cast<int> (ret);
 }
 
+bool LedgerMaster::isCaughtUp(std::string& reason)
+{
+    if (getValidatedLedgerAge() > 180)
+    {
+        reason = "No recently-validated ledger";
+        return false;
+    }
+    boost::recursive_mutex::scoped_lock ml (mLock);
+    if (!mValidLedger || !mPubLedger)
+    {
+        reason = "No published ledger";
+        return false;
+    }
+    if (mValidLedger->getLedgerSeq() > (mPubLedger->getLedgerSeq() + 3))
+    {
+        reason = "Published ledger lags validated ledger";
+        return false;
+    }
+    return true;
+}
+
 void LedgerMaster::addHeldTransaction (Transaction::ref transaction)
 {
     // returns true if transaction was added

src/cpp/ripple/LedgerMaster.h

@@ -64,6 +64,7 @@ public:
         return mPubLedger;
     }
     int getValidatedLedgerAge ();
+    bool isCaughtUp(std::string& reason);
 
     TER doTransaction (SerializedTransaction::ref txn, TransactionEngineParams params, bool& didApply);
 

src/cpp/ripple/NetworkOPs.cpp

@@ -246,14 +246,14 @@ void NetworkOPs::runTransactionQueue ()
 
             if (isTemMalformed (r)) // malformed, cache bad
                 getApp().getHashRouter ().setFlag (txn->getID (), SF_BAD);
-            else if (isTelLocal (r) || isTerRetry (r)) // can be retried
+//            else if (isTelLocal (r) || isTerRetry (r)) // can be retried
-                getApp().getHashRouter ().setFlag (txn->getID (), SF_RETRY);
+//                getApp().getHashRouter ().setFlag (txn->getID (), SF_RETRY);
 
 
             if (isTerRetry (r))
             {
                 // transaction should be held
-                WriteLog (lsDEBUG, NetworkOPs) << "Transaction should be held: " << r;
+                WriteLog (lsDEBUG, NetworkOPs) << "QTransaction should be held: " << r;
                 dbtx->setStatus (HELD);
                 getApp().getMasterTransaction ().canonicalize (dbtx, true);
                 mLedgerMaster->addHeldTransaction (dbtx);
@@ -261,27 +261,29 @@ void NetworkOPs::runTransactionQueue ()
             else if (r == tefPAST_SEQ)
             {
                 // duplicate or conflict
-                WriteLog (lsINFO, NetworkOPs) << "Transaction is obsolete";
+                WriteLog (lsINFO, NetworkOPs) << "QTransaction is obsolete";
                 dbtx->setStatus (OBSOLETE);
             }
             else if (r == tesSUCCESS)
             {
-                WriteLog (lsINFO, NetworkOPs) << "Transaction is now included in open ledger";
+                WriteLog (lsINFO, NetworkOPs) << "QTransaction is now included in open ledger";
                 dbtx->setStatus (INCLUDED);
                 getApp().getMasterTransaction ().canonicalize (dbtx, true);
             }
             else
             {
-                WriteLog (lsDEBUG, NetworkOPs) << "Status other than success " << r;
+                WriteLog (lsDEBUG, NetworkOPs) << "QStatus other than success " << r;
                 dbtx->setStatus (INVALID);
             }
 
-            if (didApply || (mMode != omFULL))
+//            if (didApply || (mMode != omFULL))
+            if (didApply)
             {
                 std::set<uint64> peers;
 
                 if (getApp().getHashRouter ().swapSet (txn->getID (), peers, SF_RELAYED))
                 {
+                    WriteLog (lsDEBUG, NetworkOPs) << "relaying";
                     protocol::TMTransaction tx;
                     Serializer s;
                     dbtx->getSTransaction ()->add (s);
@@ -292,6 +294,8 @@ void NetworkOPs::runTransactionQueue ()
                     PackedMessage::pointer packet = boost::make_shared<PackedMessage> (tx, protocol::mtTRANSACTION);
                     getApp().getPeers ().relayMessageBut (peers, packet);
                 }
+                else
+                    WriteLog(lsDEBUG, NetworkOPs) << "recently relayed";
             }
 
             txn->doCallbacks (r);
@@ -339,8 +343,8 @@ Transaction::pointer NetworkOPs::processTransaction (Transaction::pointer trans,
 
     if (isTemMalformed (r)) // malformed, cache bad
         getApp().getHashRouter ().setFlag (trans->getID (), SF_BAD);
-    else if (isTelLocal (r) || isTerRetry (r)) // can be retried
+//    else if (isTelLocal (r) || isTerRetry (r)) // can be retried
-        getApp().getHashRouter ().setFlag (trans->getID (), SF_RETRY);
+//        getApp().getHashRouter ().setFlag (trans->getID (), SF_RETRY);
 
 #ifdef BEAST_DEBUG
 

src/cpp/ripple/RPCHandler.cpp

@@ -3680,9 +3680,7 @@ Json::Value RPCHandler::doCommand (const Json::Value& params, int iRole, LoadTyp
         return rpcError (rpcNO_NETWORK);
     }
 
-    // XXX Should verify we have a current ledger.
+    if ((commandsA[i].iOptions & optCurrent) && (getApp().getLedgerMaster().getValidatedLedgerAge() > 60))
-
-    if ((commandsA[i].iOptions & optCurrent) && false)
     {
         return rpcError (rpcNO_CURRENT);
     }

src/cpp/ripple/TransactionEngine.cpp

@@ -146,7 +146,7 @@ TER TransactionEngine::applyTransaction (const SerializedTransaction& txn, Trans
             }
         }
         else
-            WriteLog (lsDEBUG, TransactionEngine) << "Not applying transaction";
+            WriteLog (lsDEBUG, TransactionEngine) << "Not applying transaction " << txID;
 
         if (didApply)
         {

src/cpp/ripple/ripple_Application.cpp

@@ -843,14 +843,6 @@ bool serverOkay (std::string& reason)
     if (!theConfig.ELB_SUPPORT)
         return true;
 
-    /*
-    if (!theApp)
-    {
-        reason = "Server has not started";
-        return false;
-    }
-    */
-
     if (getApp().isShutdown ())
     {
         reason = "Server is shutting down";
@@ -869,6 +861,9 @@ bool serverOkay (std::string& reason)
         return false;
     }
 
+    if (!getApp().getLedgerMaster().isCaughtUp(reason))
+        return false;
+
     if (getApp().getFeeTrack ().isLoaded ())
     {
         reason = "Too much load";

src/cpp/ripple/ripple_IHashRouter.h

@@ -31,7 +31,7 @@ public:
     //
     static inline int getDefaultHoldTime ()
     {
-        return 120;
+        return 300;
     }
 
     // VFALCO TODO rename the parameter to entryHoldTimeInSeconds

src/cpp/ripple/ripple_LoadManager.cpp

@@ -195,6 +195,8 @@ private:
 
         // We do it this way in case we want to add exponential decay later
         int now = UptimeTimer::getInstance ().getElapsedSeconds ();
+
+        boost::mutex::scoped_lock sl (mLock);
         canonicalize (source, now);
         source.mBalance += credits;
 
@@ -204,7 +206,8 @@ private:
         if (source.isPrivileged ()) // privileged sources never warn/cutoff
             return false;
 
-        if ((source.mBalance >= mDebitLimit) && (source.mLastWarning == now)) // no need to warn
+        if ( (source.mBalance >= mDebitWarn) ||
+            ((source.mBalance >= mDebitLimit) && (source.mLastWarning == now)))
             return false;
 
         return true;

src/cpp/ripple/ripple_Peer.cpp

@@ -197,7 +197,7 @@ PeerImp::PeerImp (boost::asio::io_service& io_service, boost::asio::ssl::context
     mCluster (false),
     mPeerId (peerID),
     mPrivate (false),
-    mLoad (""),
+    mLoad (std::string()),
     mMinLedger (0),
     mMaxLedger (0),
     mSocketSsl (io_service, ctx),
@@ -978,6 +978,8 @@ void PeerImp::recvHello (protocol::TMHello& packet)
         {
             mCluster = true;
             mLoad.setPrivileged ();
+            if (!mNodeName.empty())
+                mLoad.rename (mNodeName);
             WriteLog (lsINFO, Peer) << "Cluster connection to \"" << (mNodeName.empty () ? getIP () : mNodeName)
                                     << "\" established";
         }
@@ -1104,10 +1106,11 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast
         Serializer s (packet.rawtransaction ());
         SerializerIterator sit (s);
         SerializedTransaction::pointer stx = boost::make_shared<SerializedTransaction> (boost::ref (sit));
+        uint256 txID = stx->getTransactionID();
 
         int flags;
 
-        if (! getApp().getHashRouter ().addSuppressionPeer (stx->getTransactionID (), mPeerId, flags))
+        if (! getApp().getHashRouter ().addSuppressionPeer (txID, mPeerId, flags))
         {
             // we have seen this transaction recently
             if (isSetBit (flags, SF_BAD))
@@ -1120,12 +1123,24 @@ void PeerImp::recvTransaction (protocol::TMTransaction& packet, ScopedLock& Mast
                 return;
         }
 
-        WriteLog (lsDEBUG, Peer) << "Got new transaction from peer";
+        if (getApp().getMasterTransaction().fetch(txID, true))
+        {
+            WriteLog (lsDEBUG, Peer) << "Peer " << getDisplayName() << " send old TX " << txID;
+            applyLoadCharge (LT_InvalidRequest);
+            return;
+        }
+
+        WriteLog (lsDEBUG, Peer) << "Got new transaction from peer " << getDisplayName () << " : " << txID;
 
         if (mCluster)
             flags |= SF_TRUSTED | SF_SIGGOOD;
 
-        getApp().getJobQueue ().addJob (jtTRANSACTION, "recvTransction->checkTransaction",
+        if (getApp().getJobQueue().getJobCount(jtTRANSACTION) > 100)
+            WriteLog(lsINFO, Peer) << "Transaction queue is full";
+        else if (getApp().getLedgerMaster().getValidatedLedgerAge() > 240)
+            WriteLog(lsINFO, Peer) << "No new transactions until synchronized";
+        else
+            getApp().getJobQueue ().addJob (jtTRANSACTION, "recvTransction->checkTransaction",
                                        BIND_TYPE (&checkTransaction, P_1, flags, stx, boost::weak_ptr<Peer> (shared_from_this ())));
 
 #ifndef TRUST_NETWORK
@@ -2213,11 +2228,28 @@ void PeerImp::sendGetPeers ()
 
 void PeerImp::applyLoadCharge (LoadType loadType)
 {
+    // IMPLEMENTATION IS INCOMPLETE
+
+    // VFALCO TODO This needs to completed before open sourcing.
+
     if (getApp().getLoadManager ().applyLoadCharge (mLoad, loadType))
     {
-        // UNIMPLEMENTED
+        if (mCluster)
-
+        {
-        // VFALCO TODO This needs to implemented before open sourcing.
+            WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " load from cluster";
+        }
+        else if (getApp().getLoadManager ().shouldCutoff(mLoad))
+        {
+            WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " should cutoff";
+        }
+        else if (getApp().getLoadManager ().shouldWarn (mLoad))
+        {
+            WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " load warning";
+        }
+        else
+        {
+            WriteLog (lsWARNING, Peer) << "aLC: " << getDisplayName() << " cannot figure out";
+        }
     }
 }
 

src/cpp/ripple/ripple_RippleCalc.cpp

@@ -49,7 +49,7 @@ TER RippleCalc::calcNodeAdvance (
 
     TER             terResult       = tesSUCCESS;
 
-    WriteLog (lsDEBUG, RippleCalc) << "calcNodeAdvance";
+    WriteLog (lsDEBUG, RippleCalc) << "calcNodeAdvance: TakerPays:" << saTakerPays << " TakerGets:" << saTakerGets;
 
     int loopCount = 0;
 
@@ -478,6 +478,9 @@ TER RippleCalc::calcNodeDeliverRev (
                                       % saOutPassAct
                                       % saOutPlusFees);
 
+        if (saInPassReq > saTakerPays)
+            saInPassReq = saTakerPays;
+
         if (!saInPassReq)
         {
             // After rounding did not want anything.
@@ -690,7 +693,7 @@ TER RippleCalc::calcNodeDeliverFwd (
             STAmount    saInFunded      = STAmount::mulRound (saOutPassFunded, saOfrRate, saTakerPays, true); // Offer maximum in - Limited by by payout.
             STAmount    saInTotal       = STAmount::mulRound (saInFunded, saInFeeRate, true);               // Offer maximum in with fees.
             STAmount    saInSum         = std::min (saInTotal, saInReq - saInAct - saInFees);               // In limited by remaining.
-            STAmount    saInPassAct     = STAmount::divRound (saInSum, saInFeeRate, true);                  // In without fees.
+            STAmount    saInPassAct     = std::min (saTakerPays, STAmount::divRound (saInSum, saInFeeRate, true));     // In without fees.
             STAmount    saOutPassMax    = std::min (saOutPassFunded, STAmount::divRound (saInPassAct, saOfrRate, saTakerGets, true)); // Out limited by in remaining.
 
             STAmount    saInPassFeesMax = saInSum - saInPassAct;
@@ -789,7 +792,7 @@ TER RippleCalc::calcNodeDeliverFwd (
 
                     assert (saOutPassAct < saOutPassMax);
 
-                    saInPassAct     = STAmount::mulRound (saOutPassAct, saOfrRate, saInReq, true);
+                    saInPassAct     = std::min (saTakerPays, STAmount::mulRound (saOutPassAct, saOfrRate, saInReq, true));
                     saInPassFees    = std::min (saInPassFeesMax, STAmount::mulRound (saInPassAct, saInFeeRate, true));
                 }
 
@@ -2365,6 +2368,7 @@ void TransactionEngine::calcOfferBridgeNext (
         {
             // Offer is expired.
             WriteLog (lsINFO, RippleCalc) << "calcOfferFirst: encountered expired offer";
+            musUnfundedFound.insert(uOfferIndex);
         }
         else
         {

src/cpp/ripple/ripple_SHAMap.cpp

@@ -889,7 +889,7 @@ bool SHAMap::fetchRoot (uint256 const& hash, SHAMapSyncFilter* filter)
     SHAMapTreeNode::pointer newRoot = fetchNodeExternalNT(SHAMapNode(), hash);
     if (newRoot)
     	root = newRoot;
-	else
+    else
     {
         Blob nodeData;
 

src/cpp/ripple/ripple_SHAMapSync.cpp

@@ -56,7 +56,7 @@ void SHAMap::getMissingNodes (std::vector<SHAMapNode>& nodeIDs, std::vector<uint
 
                     if (!d)
                     {
-                        // node is not in the map
+                        // node is not in the database
                         nodeIDs.push_back (childID);
                         hashes.push_back (childHash);
 

src/cpp/ripple/ripple_SHAMapTreeNode.cpp

@@ -356,12 +356,10 @@ void SHAMapTreeNode::addRaw (Serializer& s, SHANodeFormat format)
 
 bool SHAMapTreeNode::setItem (SHAMapItem::ref i, TNType type)
 {
-    uint256 hash = getNodeHash ();
     mType = type;
     mItem = i;
     assert (isLeaf ());
-    updateHash ();
+    return updateHash ();
-    return getNodeHash () != hash;
 }
 
 SHAMapItem::pointer SHAMapTreeNode::getItem () const

src/cpp/ripple/ripple_SHAMapTreeNode.h

@@ -124,7 +124,7 @@ public:
         return !!mItem;
     }
     SHAMapItem::ref peekItem ()
-    {
+    { // CAUTION: Do not modify the item
         return mItem;
     }
     SHAMapItem::pointer getItem () const;