Revert "Don't save unvalidated transactions to any transaction database."

This reverts commit 28a7e58fd4.

modules/ripple_app/ledger/Ledger.cpp

@@ -15,7 +15,7 @@ Ledger::Ledger (const RippleAddress& masterID, uint64 startAmount)
     , mCloseFlags (0)
     , mClosed (false)
     , mValidHash (false)
-    , mValidated (false)
+    , mAccepted (false)
     , mImmutable (false)
     , mTransactionMap (boost::make_shared <SHAMap> (smtTRANSACTION))
     , mAccountStateMap (boost::make_shared <SHAMap> (smtSTATE))
@@ -59,7 +59,7 @@ Ledger::Ledger (uint256 const& parentHash,
     , mCloseFlags (closeFlags)
     , mClosed (false)
     , mValidHash (false)
-    , mValidated (false)
+    , mAccepted (false)
     , mImmutable (true)
     , mTransactionMap (boost::make_shared <SHAMap> (smtTRANSACTION, transHash))
     , mAccountStateMap (boost::make_shared <SHAMap> (smtSTATE, accountHash))
@@ -98,7 +98,7 @@ Ledger::Ledger (Ledger& ledger,
     , mCloseFlags (ledger.mCloseFlags)
     , mClosed (ledger.mClosed)
     , mValidHash (false)
-    , mValidated (ledger.mValidated)
+    , mAccepted (ledger.mAccepted)
     , mImmutable (!isMutable)
     , mTransactionMap (ledger.mTransactionMap->snapShot (isMutable))
     , mAccountStateMap (ledger.mAccountStateMap->snapShot (isMutable))
@@ -117,7 +117,7 @@ Ledger::Ledger (bool /* dummy */,
     , mCloseFlags (0)
     , mClosed (false)
     , mValidHash (false)
-    , mValidated (true)
+    , mAccepted (false)
     , mImmutable (false)
     , mTransactionMap (boost::make_shared <SHAMap> (smtTRANSACTION))
     , mAccountStateMap (prevLedger.mAccountStateMap->snapShot (true))
@@ -149,7 +149,7 @@ Ledger::Ledger (Blob const& rawLedger,
                 bool hasPrefix)
     : mClosed (false)
     , mValidHash (false)
-    , mValidated (false)
+    , mAccepted (false)
     , mImmutable (true)
 {
     Serializer s (rawLedger);
@@ -160,7 +160,7 @@ Ledger::Ledger (Blob const& rawLedger,
 }
 
 Ledger::Ledger (const std::string& rawLedger, bool hasPrefix) :
-    mClosed (false), mValidHash (false), mValidated (false), mImmutable (true)
+    mClosed (false), mValidHash (false), mAccepted (false), mImmutable (true)
 {
     Serializer s (rawLedger);
     setRaw (s, hasPrefix);
@@ -243,10 +243,11 @@ void Ledger::addRaw (Serializer& s) const
 void Ledger::setAccepted (uint32 closeTime, int closeResolution, bool correctCloseTime)
 {
     // used when we witnessed the consensus
-    assert (mClosed);
+    assert (mClosed && !mAccepted);
     mCloseTime = correctCloseTime ? roundCloseTime (closeTime, closeResolution) : closeTime;
     mCloseResolution = closeResolution;
     mCloseFlags = correctCloseTime ? 0 : sLCF_NoConsensusTime;
+    mAccepted = true;
     setImmutable ();
 }
 
@@ -257,6 +258,7 @@ void Ledger::setAccepted ()
     if ((mCloseFlags & sLCF_NoConsensusTime) == 0)
         mCloseTime = roundCloseTime (mCloseTime, mCloseResolution);
 
+    mAccepted = true;
     setImmutable ();
 }
 
@@ -497,9 +499,9 @@ uint256 Ledger::getHash ()
     return mHash;
 }
 
-void Ledger::saveValidatedLedger (Job&)
+void Ledger::saveAcceptedLedger (Job&, bool fromConsensus)
 {
-    WriteLog (lsTRACE, Ledger) << "saveValidatedLedger ";
+    WriteLog (lsTRACE, Ledger) << "saveAcceptedLedger " << (fromConsensus ? "fromConsensus " : "fromAcquire ") << getLedgerSeq ();
     static boost::format deleteLedger ("DELETE FROM Ledgers WHERE LedgerSeq = %u;");
     static boost::format deleteTrans1 ("DELETE FROM Transactions WHERE LedgerSeq = %u;");
     static boost::format deleteTrans2 ("DELETE FROM AccountTransactions WHERE LedgerSeq = %u;");
@@ -520,7 +522,7 @@ void Ledger::saveValidatedLedger (Job&)
     if (getAccountHash () != mAccountStateMap->getHash ())
     {
         WriteLog (lsFATAL, Ledger) << "sAL: " << getAccountHash () << " != " << mAccountStateMap->getHash ();
-        WriteLog (lsFATAL, Ledger) << "saveValidatedLedger: seq=" << mLedgerSeq;
+        WriteLog (lsFATAL, Ledger) << "saveAcceptedLedger: seq=" << mLedgerSeq << ", fromcons=" << fromConsensus;
         assert (false);
     }
 
@@ -605,6 +607,9 @@ void Ledger::saveValidatedLedger (Job&)
                 mCloseResolution % mCloseFlags % mAccountHash.GetHex () % mTransHash.GetHex ()));
     }
 
+    if (!fromConsensus && (getConfig ().NODE_SIZE < 2)) // tiny or small
+        dropCache ();
+
     if (getApp().getJobQueue ().getJobCountTotal (jtPUBOLDLEDGER) < 2)
         getApp().getLedgerMaster ().resumeAcquiring ();
     else
@@ -728,9 +733,11 @@ Ledger::pointer Ledger::getSQL (const std::string& sql)
     if (!loaded)
         return Ledger::pointer ();
 
-    ret->setValidated ();
     ret->setClosed ();
 
+    if (getApp().getOPs ().haveLedger (ledgerSeq))
+        ret->setAccepted ();
+
     if (ret->getHash () != ledgerHash)
     {
         if (ShouldLog (lsERROR, Ledger))
@@ -794,7 +801,9 @@ void Ledger::getSQL2 (Ledger::ref ret)
 {
     ret->setClosed ();
     ret->setImmutable ();
-    ret->setValidated ();
+
+    if (getApp().getOPs ().haveLedger (ret->getLedgerSeq ()))
+        ret->setAccepted ();
 
     WriteLog (lsTRACE, Ledger) << "Loaded ledger: " << ret->getHash ().GetHex ();
 }
@@ -951,7 +960,7 @@ Json::Value Ledger::getJson (int options)
         ledger["ledger_hash"]       = mHash.GetHex ();
         ledger["transaction_hash"]  = mTransHash.GetHex ();
         ledger["account_hash"]      = mAccountHash.GetHex ();
-        ledger["validated"]         = mValidated;
+        ledger["accepted"]          = mAccepted;
         ledger["total_coins"]       = boost::lexical_cast<std::string> (mTotCoins);
 
         if (mCloseTime != 0)
@@ -1795,19 +1804,17 @@ uint32 Ledger::roundCloseTime (uint32 closeTime, uint32 closeResolution)
     return closeTime - (closeTime % closeResolution);
 }
 
-void Ledger::pendSaveValidated ()
+void Ledger::pendSave (bool fromConsensus)
 {
-    if (getApp().getHashRouter ().setFlag (getHash (), SF_SAVED))
+    if (!fromConsensus && !getApp().getHashRouter ().setFlag (getHash (), SF_SAVED))
         return;
 
-    assert (isImmutable() && isValidated());
+    assert (isImmutable ());
-
-    bool isNew = getLedgerSeq() > getApp().getLedgerMaster().getPublishedLedger()->getLedgerSeq();
 
     getApp().getJobQueue ().addJob (
-            isNew ? jtPUBLEDGER : jtPUBOLDLEDGER,
+        fromConsensus ? jtPUBLEDGER : jtPUBOLDLEDGER,
-            isNew ? "Ledger::pendSave" : "Ledger::pendOldSave",
+        fromConsensus ? "Ledger::pendSave" : "Ledger::pendOldSave",
-            BIND_TYPE (&Ledger::saveValidatedLedger, shared_from_this (), P_1));
+        BIND_TYPE (&Ledger::saveAcceptedLedger, shared_from_this (), P_1, fromConsensus));
 
 }
 

modules/ripple_app/ledger/Ledger.h

@@ -93,18 +93,14 @@ public:
     }
     void setAccepted (uint32 closeTime, int closeResolution, bool correctCloseTime);
     void setAccepted ();
-    void setValidated ()
-    {
-        mValidated = mClosed = true;
-    }
     void setImmutable ();
     bool isClosed ()
     {
         return mClosed;
     }
-    bool isValidated ()
+    bool isAccepted ()
     {
-        return mValidated;
+        return mAccepted;
     }
     bool isImmutable ()
     {
@@ -232,7 +228,7 @@ public:
     static uint256 getHashByIndex (uint32 index);
     static bool getHashesByIndex (uint32 index, uint256 & ledgerHash, uint256 & parentHash);
     static std::map< uint32, std::pair<uint256, uint256> > getHashesByIndex (uint32 minSeq, uint32 maxSeq);
-    void pendSaveValidated ();
+    void pendSave (bool fromConsensus);
 
     // next/prev function
     SLE::pointer getSLE (uint256 const & uHash); // SLE is mutable
@@ -430,7 +426,7 @@ protected:
     // returned SLE is immutable
     SLE::pointer getASNodeI (uint256 const & nodeID, LedgerEntryType let);
 
-    void saveValidatedLedger (Job&);
+    void saveAcceptedLedger (Job&, bool fromConsensus);
 
     void updateFees ();
 
@@ -453,7 +449,7 @@ private:
     uint32      mParentCloseTime;   // when the previous ledger closed
     int         mCloseResolution;   // the resolution for this ledger close time (2-120 seconds)
     uint32      mCloseFlags;        // flags indicating how this ledger close took place
-    bool        mClosed, mValidHash, mValidated, mImmutable;
+    bool        mClosed, mValidHash, mAccepted, mImmutable;
 
     uint32      mReferenceFeeUnits;                 // Fee units for the reference transaction
     uint32      mReserveBase, mReserveIncrement;    // Reserve basse and increment in fee units

modules/ripple_app/ledger/LedgerMaster.cpp

@@ -105,16 +105,23 @@ void LedgerMaster::pushLedger (Ledger::pointer newLedger)
     mFinalizedLedger = mCurrentLedger;
     mCurrentLedger = newLedger;
     mEngine.setLedger (newLedger);
-    mLedgerHistory.addLedger (newLedger);
 }
 
 void LedgerMaster::pushLedger (Ledger::pointer newLCL, Ledger::pointer newOL, bool fromConsensus)
 {
-    assert (newLCL->isClosed () && newLCL->isImmutable());
+    assert (newLCL->isClosed () && newLCL->isAccepted ());
-    assert (!newOL->isClosed ());
+    assert (!newOL->isClosed () && !newOL->isAccepted ());
 
     boost::recursive_mutex::scoped_lock ml (mLock);
 
+    if (newLCL->isAccepted ())
+    {
+        assert (newLCL->isClosed ());
+        assert (newLCL->isImmutable ());
+        mLedgerHistory.addAcceptedLedger (newLCL, fromConsensus);
+        WriteLog (lsINFO, LedgerMaster) << "StashAccepted: " << newLCL->getHash ();
+    }
+
     {
         boost::recursive_mutex::scoped_lock ml (mLock);
         mFinalizedLedger = newLCL;
@@ -145,6 +152,9 @@ void LedgerMaster::switchLedgers (Ledger::pointer lastClosed, Ledger::pointer cu
 void LedgerMaster::storeLedger (Ledger::pointer ledger)
 {
     mLedgerHistory.addLedger (ledger);
+
+    if (ledger->isAccepted ())
+        mLedgerHistory.addAcceptedLedger (ledger, false);
 }
 
 void LedgerMaster::forceValid (Ledger::pointer ledger)
@@ -328,10 +338,7 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const&
         Ledger::pointer lgr = mMissingLedger->getLedger ();
 
         if (lgr && (lgr->getLedgerSeq () == ledgerSeq))
-        {
-            lgr->setValidated();
             missingAcquireComplete (mMissingLedger);
-        }
 
         mMissingLedger.reset ();
         return true;
@@ -374,8 +381,8 @@ bool LedgerMaster::acquireMissingLedger (Ledger::ref origLedger, uint256 const&
                     if (acq && acq->isComplete ())
                     {
                         acq->getLedger ()->setAccepted ();
-                        setValidatedLedger (acq->getLedger ());
+                        setFullLedger (acq->getLedger ());
-                        mLedgerHistory.addValidatedLedger (acq->getLedger ());
+                        mLedgerHistory.addAcceptedLedger (acq->getLedger (), false);
                     }
                     else
                         ++fetchCount;
@@ -439,8 +446,9 @@ void LedgerMaster::missingAcquireComplete (InboundLedger::pointer acq)
 
     if (acq->isComplete ())
     {
-        setValidatedLedger (acq->getLedger ());
+        acq->getLedger ()->setAccepted ();
-        mLedgerHistory.addValidatedLedger (acq->getLedger ());
+        setFullLedger (acq->getLedger ());
+        mLedgerHistory.addAcceptedLedger (acq->getLedger (), false);
     }
 }
 
@@ -533,7 +541,7 @@ void LedgerMaster::fixMismatch (Ledger::ref ledger)
     CondLog (invalidate != 0, lsWARNING, LedgerMaster) << "All " << invalidate << " prior ledgers invalidated";
 }
 
-void LedgerMaster::setValidatedLedger (Ledger::pointer ledger)
+void LedgerMaster::setFullLedger (Ledger::pointer ledger)
 {
     // A new ledger has been accepted as part of the trusted chain
     WriteLog (lsDEBUG, LedgerMaster) << "Ledger " << ledger->getLedgerSeq () << " accepted :" << ledger->getHash ();
@@ -544,11 +552,13 @@ void LedgerMaster::setValidatedLedger (Ledger::pointer ledger)
 
     boost::recursive_mutex::scoped_lock ml (mLock);
 
-    ledger->setValidated();
-
     mCompleteLedgers.setValue (ledger->getLedgerSeq ());
 
-    ledger->pendSaveValidated ();
+    if (Ledger::getHashByIndex (ledger->getLedgerSeq ()) != ledger->getHash ())
+    {
+        ledger->pendSave (false);
+        return;
+    }
 
     if ((ledger->getLedgerSeq () != 0) && mCompleteLedgers.hasValue (ledger->getLedgerSeq () - 1))
     {
@@ -814,7 +824,7 @@ void LedgerMaster::pubThread ()
         BOOST_FOREACH (Ledger::ref l, ledgers)
         {
             WriteLog (lsDEBUG, LedgerMaster) << "Publishing ledger " << l->getLedgerSeq ();
-            setValidatedLedger (l);
+            setFullLedger (l); // OPTIMIZEME: This is actually more work than we need to do
             getApp().getOPs ().pubLedger (l);
             published = true;
         }

modules/ripple_app/ledger/LedgerMaster.h

@@ -90,7 +90,7 @@ public:
     void storeLedger (Ledger::pointer);
     void forceValid (Ledger::pointer);
 
-    void setValidatedLedger (Ledger::pointer ledger);
+    void setFullLedger (Ledger::pointer ledger);
 
     void switchLedgers (Ledger::pointer lastClosed, Ledger::pointer newCurrent);
 

modules/ripple_app/ledger/ripple_LedgerHistory.cpp

@@ -27,16 +27,19 @@ void LedgerHistory::addLedger (Ledger::pointer ledger)
     mLedgersByHash.canonicalize (ledger->getHash (), ledger, true);
 }
 
-void LedgerHistory::addValidatedLedger (Ledger::pointer ledger)
+void LedgerHistory::addAcceptedLedger (Ledger::pointer ledger, bool fromConsensus)
 {
-    assert (ledger && ledger->isValidated () && ledger->isImmutable ());
+    assert (ledger && ledger->isAccepted () && ledger->isImmutable ());
     assert (ledger->peekAccountStateMap ()->getHash ().isNonZero ());
     uint256 h (ledger->getHash ());
     boost::recursive_mutex::scoped_lock sl (mLedgersByHash.peekMutex ());
-    mLedgersByHash.canonicalize (h, ledger);
+    mLedgersByHash.canonicalize (h, ledger, true);
+    assert (ledger);
+    assert (ledger->isAccepted ());
+    assert (ledger->isImmutable ());
     mLedgersByIndex[ledger->getLedgerSeq ()] = ledger->getHash ();
 
-    ledger->pendSaveValidated ();
+    ledger->pendSave (fromConsensus);
 }
 
 uint256 LedgerHistory::getLedgerHash (uint32 index)
@@ -123,7 +126,7 @@ Ledger::pointer LedgerHistory::canonicalizeLedger (Ledger::pointer ledger, bool
     boost::recursive_mutex::scoped_lock sl (mLedgersByHash.peekMutex ());
     mLedgersByHash.canonicalize (h, ledger);
 
-    if (ledger->isValidated ())
+    if (ledger->isAccepted ())
         mLedgersByIndex[ledger->getLedgerSeq ()] = ledger->getHash ();
 
     return ledger;

modules/ripple_app/ledger/ripple_LedgerHistory.h

@@ -15,7 +15,7 @@ public:
 
     void addLedger (Ledger::pointer ledger);
 
-    void addValidatedLedger (Ledger::pointer ledger);
+    void addAcceptedLedger (Ledger::pointer ledger, bool fromConsensus);
 
     float getCacheHitRate ()
     {

modules/ripple_app/misc/NetworkOPs.cpp

@@ -1272,7 +1272,7 @@ NetworkOPs::getAccountTxs (const RippleAddress& account, int32 minLedger, int32
                 WriteLog(lsWARNING, NetworkOPs) << "Recovering ledger " << seq << ", txn " << txn->getID();
                 Ledger::pointer ledger = getLedgerBySeq(seq);
                 if (ledger)
-                    ledger->pendSaveValidated();
+                    ledger->pendSave(false);
             }
 
             TransactionMetaSet::pointer meta = boost::make_shared<TransactionMetaSet> (txn->getID (), txn->getLedger (), rawMeta.getData ());