From 62516ef07fc63c17649057e36c7fa238acb6b34f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Tue, 7 Jan 2014 20:52:03 -0800 Subject: [PATCH] Refactor TaggedCache --- Builds/VisualStudio2012/RippleD.vcxproj | 1 - .../VisualStudio2012/RippleD.vcxproj.filters | 3 - src/ripple/types/api/UInt256.h | 17 + src/ripple/types/api/base_uint.h | 7 +- src/ripple_app/ledger/AcceptedLedger.cpp | 8 +- src/ripple_app/ledger/AcceptedLedger.h | 2 +- src/ripple_app/ledger/LedgerHistory.cpp | 69 +- src/ripple_app/ledger/LedgerHistory.h | 16 +- src/ripple_app/main/Application.cpp | 13 +- src/ripple_app/main/Application.h | 4 +- src/ripple_app/misc/NetworkOPs.cpp | 6 +- src/ripple_app/misc/Validations.cpp | 7 +- src/ripple_app/ripple_app.h | 13 +- src/ripple_app/shamap/SHAMap.cpp | 6 +- src/ripple_app/shamap/SHAMap.h | 5 +- src/ripple_app/shamap/SHAMapNode.h | 32 + src/ripple_app/shamap/SHAMapSyncFilters.cpp | 8 +- src/ripple_app/shamap/SHAMapSyncFilters.h | 8 +- src/ripple_app/tx/TransactionAcquire.cpp | 5 +- src/ripple_app/tx/TransactionMaster.cpp | 12 +- src/ripple_app/tx/TransactionMaster.h | 2 +- src/ripple_basics/containers/BlackList.h | 213 ----- src/ripple_basics/containers/TaggedCache.cpp | 136 ++- src/ripple_basics/containers/TaggedCache.h | 796 +++++++++--------- src/ripple_basics/ripple_basics.h | 9 +- src/ripple_core/nodestore/impl/DatabaseImp.h | 9 +- 26 files changed, 700 insertions(+), 707 deletions(-) delete mode 100644 src/ripple_basics/containers/BlackList.h diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index e34c07ff9b..c9436d2d84 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -2452,7 +2452,6 @@ - diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 25fa02abdb..ca994b33b8 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -2427,9 +2427,6 @@ [1] Ripple\types\api - - [2] Old Ripple\ripple_basics\containers - [2] Old Ripple\ripple_core\functional diff --git a/src/ripple/types/api/UInt256.h b/src/ripple/types/api/UInt256.h index b0b77c4572..3dcc55a9e6 100644 --- a/src/ripple/types/api/UInt256.h +++ b/src/ripple/types/api/UInt256.h @@ -27,6 +27,8 @@ #include "base_uint.h" +#include + namespace ripple { class uint256 : public base_uint256 @@ -343,4 +345,19 @@ extern std::size_t hash_value (uint256 const& ); } +//------------------------------------------------------------------------------ + +namespace std { + +template <> +struct hash : std::hash > +{ + typedef std::hash > Base; + // VFALCO NOTE broken in vs2012 + //using Base::Base; // inherit ctors + +}; + +}; + #endif diff --git a/src/ripple/types/api/base_uint.h b/src/ripple/types/api/base_uint.h index 629bb83fd7..c0a2fe9d4a 100644 --- a/src/ripple/types/api/base_uint.h +++ b/src/ripple/types/api/base_uint.h @@ -25,6 +25,8 @@ #ifndef RIPPLE_TYPES_BASE_UINT_H_INCLUDED #define RIPPLE_TYPES_BASE_UINT_H_INCLUDED +#include + namespace ripple { class uint128; @@ -499,11 +501,8 @@ std::ostream& operator<< (std::ostream& out, const base_uint& u) namespace std { -template -struct hash; - /** Specialization for hash. */ -template +template struct hash > { public: diff --git a/src/ripple_app/ledger/AcceptedLedger.cpp b/src/ripple_app/ledger/AcceptedLedger.cpp index a0a39689db..f099baf565 100644 --- a/src/ripple_app/ledger/AcceptedLedger.cpp +++ b/src/ripple_app/ledger/AcceptedLedger.cpp @@ -17,7 +17,13 @@ */ //============================================================================== -TaggedCacheType AcceptedLedger::s_cache ("AcceptedLedger", 4, 60); +// VFALCO TODO Remove this global and make it a member of the App +// Use a dependency injection to give AcceptedLedger access. +// +TaggedCacheType AcceptedLedger::s_cache ( + "AcceptedLedger", 4, 60, + get_abstract_clock (), + LogPartition::getJournal ()); AcceptedLedger::AcceptedLedger (Ledger::ref ledger) : mLedger (ledger) { diff --git a/src/ripple_app/ledger/AcceptedLedger.h b/src/ripple_app/ledger/AcceptedLedger.h index 9dba942e25..9b59cbc01c 100644 --- a/src/ripple_app/ledger/AcceptedLedger.h +++ b/src/ripple_app/ledger/AcceptedLedger.h @@ -83,7 +83,7 @@ private: void insert (AcceptedLedgerTx::ref); private: - static TaggedCacheType s_cache; + static TaggedCacheType s_cache; Ledger::pointer mLedger; map_t mMap; diff --git a/src/ripple_app/ledger/LedgerHistory.cpp b/src/ripple_app/ledger/LedgerHistory.cpp index 901d5a31a5..674e54853c 100644 --- a/src/ripple_app/ledger/LedgerHistory.cpp +++ b/src/ripple_app/ledger/LedgerHistory.cpp @@ -30,8 +30,12 @@ // FIXME: Need to clean up ledgers by index at some point LedgerHistory::LedgerHistory () - : mLedgersByHash ("LedgerCache", CACHED_LEDGER_NUM, CACHED_LEDGER_AGE) - , mConsensusValidated ("ConsensusValidated", 64, 300) + : m_ledgers_by_hash ("LedgerCache", CACHED_LEDGER_NUM, CACHED_LEDGER_AGE, + get_abstract_clock (), + LogPartition::getJournal ()) + , m_consensus_validated ("ConsensusValidated", 64, 300, + get_abstract_clock (), + LogPartition::getJournal ()) { ; } @@ -41,16 +45,16 @@ void LedgerHistory::addLedger (Ledger::pointer ledger, bool validated) assert (ledger && ledger->isImmutable ()); assert (ledger->peekAccountStateMap ()->getHash ().isNonZero ()); - TaggedCache::ScopedLockType sl (mLedgersByHash.peekMutex (), __FILE__, __LINE__); + LedgersByHash::ScopedLockType sl (m_ledgers_by_hash.peekMutex ()); - mLedgersByHash.canonicalize (ledger->getHash(), ledger, true); + m_ledgers_by_hash.canonicalize (ledger->getHash(), ledger, true); if (validated) mLedgersByIndex[ledger->getLedgerSeq()] = ledger->getHash(); } uint256 LedgerHistory::getLedgerHash (uint32 index) { - TaggedCache::ScopedLockType sl (mLedgersByHash.peekMutex (), __FILE__, __LINE__); + LedgersByHash::ScopedLockType sl (m_ledgers_by_hash.peekMutex ()); std::map::iterator it (mLedgersByIndex.find (index)); if (it != mLedgersByIndex.end ()) @@ -61,17 +65,17 @@ uint256 LedgerHistory::getLedgerHash (uint32 index) Ledger::pointer LedgerHistory::getLedgerBySeq (uint32 index) { - TaggedCache::ScopedLockType sl (mLedgersByHash.peekMutex (), __FILE__, __LINE__); - std::map::iterator it (mLedgersByIndex.find (index)); - - if (it != mLedgersByIndex.end ()) { - uint256 hash = it->second; - sl.unlock (); - return getLedgerByHash (hash); - } + LedgersByHash::ScopedLockType sl (m_ledgers_by_hash.peekMutex ()); + std::map ::iterator it (mLedgersByIndex.find (index)); - sl.unlock (); + if (it != mLedgersByIndex.end ()) + { + uint256 hash = it->second; + sl.unlock (); + return getLedgerByHash (hash); + } + } Ledger::pointer ret (Ledger::loadByIndex (index)); @@ -80,16 +84,19 @@ Ledger::pointer LedgerHistory::getLedgerBySeq (uint32 index) assert (ret->getLedgerSeq () == index); - sl.lock (__FILE__, __LINE__); - assert (ret->isImmutable ()); - mLedgersByHash.canonicalize (ret->getHash (), ret); - mLedgersByIndex[ret->getLedgerSeq ()] = ret->getHash (); - return (ret->getLedgerSeq () == index) ? ret : Ledger::pointer (); + { + LedgersByHash::ScopedLockType sl (m_ledgers_by_hash.peekMutex ()); + + assert (ret->isImmutable ()); + m_ledgers_by_hash.canonicalize (ret->getHash (), ret); + mLedgersByIndex[ret->getLedgerSeq ()] = ret->getHash (); + return (ret->getLedgerSeq () == index) ? ret : Ledger::pointer (); + } } Ledger::pointer LedgerHistory::getLedgerByHash (uint256 const& hash) { - Ledger::pointer ret = mLedgersByHash.fetch (hash); + Ledger::pointer ret = m_ledgers_by_hash.fetch (hash); if (ret) { @@ -105,7 +112,7 @@ Ledger::pointer LedgerHistory::getLedgerByHash (uint256 const& hash) assert (ret->isImmutable ()); assert (ret->getHash () == hash); - mLedgersByHash.canonicalize (ret->getHash (), ret); + m_ledgers_by_hash.canonicalize (ret->getHash (), ret); assert (ret->getHash () == hash); return ret; @@ -116,10 +123,11 @@ void LedgerHistory::builtLedger (Ledger::ref ledger) LedgerIndex index = ledger->getLedgerSeq(); LedgerHash hash = ledger->getHash(); assert (!hash.isZero()); - TaggedCache::ScopedLockType sl(mConsensusValidated.peekMutex(), __FILE__, __LINE__); + ConsensusValidated::ScopedLockType sl ( + m_consensus_validated.peekMutex()); boost::shared_ptr< std::pair< LedgerHash, LedgerHash > > entry = boost::make_shared>(); - mConsensusValidated.canonicalize(index, entry, false); + m_consensus_validated.canonicalize(index, entry, false); if (entry->first != hash) { @@ -140,10 +148,11 @@ void LedgerHistory::validatedLedger (Ledger::ref ledger) LedgerIndex index = ledger->getLedgerSeq(); LedgerHash hash = ledger->getHash(); assert (!hash.isZero()); - TaggedCache::ScopedLockType sl(mConsensusValidated.peekMutex(), __FILE__, __LINE__); + ConsensusValidated::ScopedLockType sl ( + m_consensus_validated.peekMutex()); boost::shared_ptr< std::pair< LedgerHash, LedgerHash > > entry = boost::make_shared>(); - mConsensusValidated.canonicalize(index, entry, false); + m_consensus_validated.canonicalize(index, entry, false); if (entry->second != hash) { @@ -159,11 +168,11 @@ void LedgerHistory::validatedLedger (Ledger::ref ledger) } } -/** Ensure mLedgersByHash doesn't have the wrong hash for a particular index +/** Ensure m_ledgers_by_hash doesn't have the wrong hash for a particular index */ bool LedgerHistory::fixIndex (LedgerIndex ledgerIndex, LedgerHash const& ledgerHash) { - TaggedCache::ScopedLockType sl (mLedgersByHash.peekMutex (), __FILE__, __LINE__); + LedgersByHash::ScopedLockType sl (m_ledgers_by_hash.peekMutex ()); std::map::iterator it (mLedgersByIndex.find (ledgerIndex)); if ((it != mLedgersByIndex.end ()) && (it->second != ledgerHash) ) @@ -176,8 +185,6 @@ bool LedgerHistory::fixIndex (LedgerIndex ledgerIndex, LedgerHash const& ledgerH void LedgerHistory::tune (int size, int age) { - mLedgersByHash.setTargetSize (size); - mLedgersByHash.setTargetAge (age); + m_ledgers_by_hash.setTargetSize (size); + m_ledgers_by_hash.setTargetAge (age); } - -// vim:ts=4 diff --git a/src/ripple_app/ledger/LedgerHistory.h b/src/ripple_app/ledger/LedgerHistory.h index e797af2113..387f43c780 100644 --- a/src/ripple_app/ledger/LedgerHistory.h +++ b/src/ripple_app/ledger/LedgerHistory.h @@ -30,7 +30,7 @@ public: float getCacheHitRate () { - return mLedgersByHash.getHitRate (); + return m_ledgers_by_hash.getHitRate (); } Ledger::pointer getLedgerBySeq (LedgerIndex ledgerIndex); @@ -43,8 +43,8 @@ public: void sweep () { - mLedgersByHash.sweep (); - mConsensusValidated.sweep (); + m_ledgers_by_hash.sweep (); + m_consensus_validated.sweep (); } void builtLedger (Ledger::ref); @@ -53,8 +53,14 @@ public: bool fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash); private: - TaggedCacheType mLedgersByHash; - TaggedCacheType , UptimeTimerAdapter> mConsensusValidated; + typedef TaggedCacheType LedgersByHash; + + LedgersByHash m_ledgers_by_hash; + + //typedef std::pair + typedef TaggedCacheType > ConsensusValidated; + ConsensusValidated m_consensus_validated; // Maps ledger indexes to the corresponding hash. diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index 9009b9bc91..569d4538e8 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -48,6 +48,9 @@ template <> char const* LogPartition::getPartitionName () { template <> char const* LogPartition::getPartitionName () { return "Collector"; } +struct TaggedCacheLog; +template <> char const* LogPartition::getPartitionName () { return "TaggedCache"; } + // //------------------------------------------------------------------------------ @@ -73,8 +76,14 @@ public: ApplicationImp () : RootStoppable ("Application") , m_journal (LogPartition::getJournal ()) - , m_tempNodeCache ("NodeCache", 16384, 90) - , m_sleCache ("LedgerEntryCache", 4096, 120) + + , m_tempNodeCache ("NodeCache", 16384, 90, + get_abstract_clock (), + LogPartition::getJournal ()) + + , m_sleCache ("LedgerEntryCache", 4096, 120, + get_abstract_clock (), + LogPartition::getJournal ()) , m_collectorManager (CollectorManager::New ( getConfig().insightSettings, diff --git a/src/ripple_app/main/Application.h b/src/ripple_app/main/Application.h index 31081d6a40..617985a35a 100644 --- a/src/ripple_app/main/Application.h +++ b/src/ripple_app/main/Application.h @@ -48,8 +48,8 @@ class LocalCredentials; class DatabaseCon; -typedef TaggedCacheType NodeCache; -typedef TaggedCacheType SLECache; +typedef TaggedCacheType NodeCache; +typedef TaggedCacheType SLECache; class Application : public PropertyStream::Source { diff --git a/src/ripple_app/misc/NetworkOPs.cpp b/src/ripple_app/misc/NetworkOPs.cpp index 3c162253a8..9f5f49c98b 100644 --- a/src/ripple_app/misc/NetworkOPs.cpp +++ b/src/ripple_app/misc/NetworkOPs.cpp @@ -50,7 +50,9 @@ public: , mLastCloseConvergeTime (1000 * LEDGER_IDLE_INTERVAL) , mLastCloseTime (0) , mLastValidationTime (0) - , mFetchPack ("FetchPack", 65536, 45) + , mFetchPack ("FetchPack", 65536, 45, + get_abstract_clock (), + LogPartition::getJournal ()) , mFetchSeq (0) , mLastLoadBase (256) , mLastLoadFactor (256) @@ -453,7 +455,7 @@ private: SubMapType mSubTransactions; // all accepted transactions SubMapType mSubRTTransactions; // all proposed and accepted transactions - TaggedCacheType< uint256, Blob , UptimeTimerAdapter > mFetchPack; + TaggedCacheType< uint256, Blob> mFetchPack; uint32 mFetchSeq; uint32 mLastLoadBase; diff --git a/src/ripple_app/misc/Validations.cpp b/src/ripple_app/misc/Validations.cpp index d711e2e741..fa863837ce 100644 --- a/src/ripple_app/misc/Validations.cpp +++ b/src/ripple_app/misc/Validations.cpp @@ -32,7 +32,7 @@ private: typedef LockType::ScopedUnlockType ScopedUnlockType; LockType mLock; - TaggedCacheType mValidations; + TaggedCacheType mValidations; boost::unordered_map mCurrentValidations; std::vector mStaleValidations; @@ -60,7 +60,10 @@ private: public: ValidationsImp () : mLock (this, "Validations", __FILE__, __LINE__) - , mValidations ("Validations", 128, 600), mWriting (false) + , mValidations ("Validations", 128, 600, + get_abstract_clock (), + LogPartition::getJournal ()) + , mWriting (false) { mStaleValidations.reserve (512); } diff --git a/src/ripple_app/ripple_app.h b/src/ripple_app/ripple_app.h index f9c6c28ed8..126b5340df 100644 --- a/src/ripple_app/ripple_app.h +++ b/src/ripple_app/ripple_app.h @@ -62,18 +62,22 @@ // #include "peers/PackedMessage.h" -namespace ripple { - // Order matters here. If you get compile errors, // reorder the include lines until the order is correct. +namespace ripple { #include "data/Database.h" #include "data/DatabaseCon.h" #include "data/SqliteDatabase.h" #include "data/DBInit.h" - #include "shamap/SHAMapItem.h" +} + +// VFALCO NOTE Have to step outside the ripple namespace to +// get the specialization for std::hash, et. al. #include "shamap/SHAMapNode.h" + +namespace ripple { #include "shamap/SHAMapTreeNode.h" #include "shamap/SHAMapMissingNode.h" #include "shamap/SHAMapSyncFilter.h" @@ -140,12 +144,11 @@ namespace ripple { #include "tx/AccountSetTransactor.h" #include "tx/TrustSetTransactor.h" #include "tx/WalletAddTransactor.h" - +// VFALCO NOTE These contracts files are bunk #include "contracts/ScriptData.h" #include "contracts/Contract.h" #include "contracts/Interpreter.h" #include "contracts/Operation.h" - } #endif diff --git a/src/ripple_app/shamap/SHAMap.cpp b/src/ripple_app/shamap/SHAMap.cpp index 39ecfc2a22..e8d752b20d 100644 --- a/src/ripple_app/shamap/SHAMap.cpp +++ b/src/ripple_app/shamap/SHAMap.cpp @@ -59,8 +59,10 @@ SHAMap::SHAMap (SHAMapType t, uint256 const& hash, mTNByID.replace(*root, root); } -TaggedCacheType< SHAMap::TNIndex, SHAMapTreeNode, UptimeTimerAdapter> - SHAMap::treeNodeCache ("TreeNodeCache", 65536, 60); +TaggedCacheType< SHAMap::TNIndex, SHAMapTreeNode> + SHAMap::treeNodeCache ("TreeNodeCache", 65536, 60, + get_abstract_clock (), + LogPartition::getJournal ()); SHAMap::~SHAMap () { diff --git a/src/ripple_app/shamap/SHAMap.h b/src/ripple_app/shamap/SHAMap.h index 44e5348343..214d321e43 100644 --- a/src/ripple_app/shamap/SHAMap.h +++ b/src/ripple_app/shamap/SHAMap.h @@ -237,11 +237,12 @@ public: treeNodeCache.setTargetAge (age); } + typedef std::pair TNIndex; + private: static KeyCache fullBelowCache; - typedef std::pair TNIndex; - static TaggedCacheType treeNodeCache; + static TaggedCacheType treeNodeCache; void dirtyUp (std::stack& stack, uint256 const & target, uint256 prevHash); std::stack getStack (uint256 const & id, bool include_nonmatching_leaf); diff --git a/src/ripple_app/shamap/SHAMapNode.h b/src/ripple_app/shamap/SHAMapNode.h index f373800aae..45dd3edbac 100644 --- a/src/ripple_app/shamap/SHAMapNode.h +++ b/src/ripple_app/shamap/SHAMapNode.h @@ -20,6 +20,10 @@ #ifndef RIPPLE_SHAMAPNODE_H #define RIPPLE_SHAMAPNODE_H +#include + +namespace ripple { + // Identifies a node in a SHA256 hash map class SHAMapNode { @@ -127,4 +131,32 @@ inline std::ostream& operator<< (std::ostream& out, const SHAMapNode& node) return out << node.getString (); } +} + +//------------------------------------------------------------------------------ + +namespace std { + +template <> +struct hash +{ + std::size_t operator() (ripple::SHAMapNode const& value) const + { + return value.getMHash (); + } +}; + +} + +//------------------------------------------------------------------------------ + +namespace boost { + +template <> +struct hash : std::hash +{ +}; + +} + #endif diff --git a/src/ripple_app/shamap/SHAMapSyncFilters.cpp b/src/ripple_app/shamap/SHAMapSyncFilters.cpp index 26378acb94..7711d04010 100644 --- a/src/ripple_app/shamap/SHAMapSyncFilters.cpp +++ b/src/ripple_app/shamap/SHAMapSyncFilters.cpp @@ -17,7 +17,8 @@ */ //============================================================================== -ConsensusTransSetSF::ConsensusTransSetSF () +ConsensusTransSetSF::ConsensusTransSetSF (NodeCache& nodeCache) + : m_nodeCache (nodeCache) { } @@ -27,7 +28,7 @@ void ConsensusTransSetSF::gotNode (bool fromFilter, const SHAMapNode& id, uint25 if (fromFilter) return; - getApp().getTempNodeCache ().store (nodeHash, nodeData); + m_nodeCache.insert (nodeHash, nodeData); if ((type == SHAMapTreeNode::tnTRANSACTION_NM) && (nodeData.size () > 16)) { @@ -53,9 +54,10 @@ void ConsensusTransSetSF::gotNode (bool fromFilter, const SHAMapNode& id, uint25 bool ConsensusTransSetSF::haveNode (const SHAMapNode& id, uint256 const& nodeHash, Blob& nodeData) { - if (getApp().getTempNodeCache ().retrieve (nodeHash, nodeData)) + if (m_nodeCache.retrieve (nodeHash, nodeData)) return true; + // VFALCO TODO Use a dependency injection here Transaction::pointer txn = getApp().getMasterTransaction().fetch(nodeHash, false); if (txn) diff --git a/src/ripple_app/shamap/SHAMapSyncFilters.h b/src/ripple_app/shamap/SHAMapSyncFilters.h index be0b2a9ba0..59d836b950 100644 --- a/src/ripple_app/shamap/SHAMapSyncFilters.h +++ b/src/ripple_app/shamap/SHAMapSyncFilters.h @@ -28,7 +28,10 @@ class ConsensusTransSetSF : public SHAMapSyncFilter { public: - ConsensusTransSetSF (); + typedef TaggedCacheType NodeCache; + + // VFALCO TODO Use a dependency injection to get the temp node cache + ConsensusTransSetSF (NodeCache& nodeCache); // Note that the nodeData is overwritten by this call void gotNode (bool fromFilter, @@ -40,6 +43,9 @@ public: bool haveNode (SHAMapNode const& id, uint256 const& nodeHash, Blob& nodeData); + +private: + NodeCache& m_nodeCache; }; // This class is only needed on add functions diff --git a/src/ripple_app/tx/TransactionAcquire.cpp b/src/ripple_app/tx/TransactionAcquire.cpp index ed15124477..f3091dfa43 100644 --- a/src/ripple_app/tx/TransactionAcquire.cpp +++ b/src/ripple_app/tx/TransactionAcquire.cpp @@ -150,7 +150,8 @@ void TransactionAcquire::trigger (Peer::ref peer) { std::vector nodeIDs; std::vector nodeHashes; - ConsensusTransSetSF sf; + // VFALCO TODO Use a dependency injection on the temp node cache + ConsensusTransSetSF sf (getApp().getTempNodeCache ()); mMap->getMissingNodes (nodeIDs, nodeHashes, 256, &sf); if (nodeIDs.empty ()) @@ -201,7 +202,7 @@ SHAMapAddNode TransactionAcquire::takeNodes (const std::list& nodeID std::list::const_iterator nodeIDit = nodeIDs.begin (); std::list< Blob >::const_iterator nodeDatait = data.begin (); - ConsensusTransSetSF sf; + ConsensusTransSetSF sf (getApp().getTempNodeCache ()); while (nodeIDit != nodeIDs.end ()) { diff --git a/src/ripple_app/tx/TransactionMaster.cpp b/src/ripple_app/tx/TransactionMaster.cpp index 59e2d9ec4d..15a7c9c07c 100644 --- a/src/ripple_app/tx/TransactionMaster.cpp +++ b/src/ripple_app/tx/TransactionMaster.cpp @@ -17,16 +17,10 @@ */ //============================================================================== -#ifndef CACHED_TRANSACTION_NUM -#define CACHED_TRANSACTION_NUM 65536 -#endif - -#ifndef CACHED_TRANSACTION_AGE -#define CACHED_TRANSACTION_AGE 1800 -#endif - TransactionMaster::TransactionMaster () - : mCache ("TransactionCache", CACHED_TRANSACTION_NUM, CACHED_TRANSACTION_AGE) + : mCache ("TransactionCache", 65536, 1800, + get_abstract_clock (), + LogPartition::getJournal ()) { ; } diff --git a/src/ripple_app/tx/TransactionMaster.h b/src/ripple_app/tx/TransactionMaster.h index bfd9cc87f3..a8d5ead197 100644 --- a/src/ripple_app/tx/TransactionMaster.h +++ b/src/ripple_app/tx/TransactionMaster.h @@ -37,7 +37,7 @@ public: void sweep (void); private: - TaggedCacheType mCache; + TaggedCacheType mCache; }; #endif diff --git a/src/ripple_basics/containers/BlackList.h b/src/ripple_basics/containers/BlackList.h deleted file mode 100644 index 5f7ccfa01a..0000000000 --- a/src/ripple_basics/containers/BlackList.h +++ /dev/null @@ -1,213 +0,0 @@ -#ifndef RIPPLE_BLACKLIST_H_INCLUDED -#define RIPPLE_BLACKLIST_H_INCLUDED - - -template -class BlackList -{ - struct iBlackList - { - int mBalance; // Exponentially-decaying "cost" balance - int mLastUpdate; // The uptime when the balance was last decayed - - iBlackList(int now) : mBalance(0), mLastUpdate(now) - { ; } - - iBlackList() : mBalance(0), mLastUpdate(0) - { ; } - }; - -public: - - // Used for import/export of current blacklist information - typedef std::pair BlackListEntry; - typedef std::vector BlackListEntryList; - - BlackList() - { - mWhiteList.push_back("127."); - mWhiteList.push_back("10."); - mWhiteList.push_back("192.168."); - } - - // We are issuing a warning to a source, update its entry - bool doWarning(const std::string& source) - { - return chargeEntry(source, mWarnCost); - } - - // We are disconnecting a source, update its entry - bool doDisconnect(const std::string& source) - { - return chargeEntry(source, mDiscCost); - } - - // We are connecting a source and need to know if it's allowed - bool isAllowed(const std::string& source) - { - boost::mutex::scoped_lock sl(mMutex); - - iBlackList* e = findEntry(source, true); - return (e == NULL) || (e->mBalance <= (mCreditLimit * mDecaySeconds)) || isWhiteListLocked(source); - } - - // Clean up stale entries - void sweep() - { - boost::mutex::scoped_lock sl(mMutex); - int expire = Timer::getElapsedSeconds() - mStaleTime; - - typename BlackListTable::iterator it = mList.begin(); - while (it != mList.end()) - { - if (it->second.mLastUpdate < expire) - mList.erase(it++); - else - it++; - } - } - - // Synchronize blacklist data across servers - BlackListEntryList getBlackList(int cutoff) - { - boost::mutex::scoped_lock sl(mMutex); - - BlackListEntryList list; - list.reserve(mList.size()); - - int now = Timer::getElapsedSeconds(); - cutoff *= mDecaySeconds; - - typename BlackListTable::iterator it = mList.begin(); - while (it != mList.end()) - { - if (!ageEntry(now, &it->second)) - mList.erase(it++); - else if (it->second.mBalance >= cutoff) - { - list.push_back(std::make_pair(it->first, it->second.mBalance / mDecaySeconds)); - ++it; - } - else - ++it; - } - - return list; - } - - void mergeBlackList(const BlackListEntryList& list) - { // Merge our black list with another black list, presumably received from a trusted peer - boost::mutex::scoped_lock sl(mMutex); - - BOOST_FOREACH(const BlackListEntry& entry, list) - { - // Find/make an entry for us corresponding to our peer's entry - iBlackList* e = findEntry(entry.first, true); - - // Decay the value at least once to ensure we don't pass the same value - // around forever without ever decaying it - int decayValue = entry.second; - decayValue -= (decayValue + mDecaySeconds - 1) / mDecaySeconds; - - // Raise our value to the decayed peer's value - e->mBalance = std::max(e->mBalance, decayValue); - } - } - - void setWhiteList(std::vector wl) - { - boost::mutex::scoped_lock sl(mMutex); - - mWhiteList.swap(wl); - } - - bool isWhiteList(const std::string& source) - { - boost::mutex::scoped_lock sl(mMutex); - return isWhiteListLocked(source); - } - - static const int mWarnCost = 10; // The cost of being warned - static const int mDiscCost = 100; // The cost of being disconnected for abuse - static const int mRejectCost = 1; // The cost of having a connection disconnected - static const int mCreditsPerSecond = 2; // Maximum cost rate permitted continuously - static const int mCreditLimit = 1000; // Maximum cost before rejections - static const int mStaleTime = 300; // Time to purge stale entries - static const int mDecaySeconds = 32; // Exponential decay constant - -private: - - typedef std::map BlackListTable; - - BlackListTable mList; - std::vector mWhiteList; - boost::mutex mMutex; - - bool isWhiteListLocked(const std::string& source) - { - BOOST_FOREACH(const std::string& entry, mWhiteList) - { // Does this source start with the entry? - if ((source.size() >= entry.size()) && (entry.compare(0, entry.size(), source) == 0)) - return true; - } - return false; - } - - bool chargeEntry(const std::string& source, int charge) - { - boost::mutex::scoped_lock sl(mMutex); - - iBlackList* e = findEntry(source, true); - e->mBalance += charge; - return e->mBalance > (mDecaySeconds * mCreditLimit); - } - - bool ageEntry(int now, iBlackList* entry) - { - if (entry->mLastUpdate != now) - { - if ((entry->mLastUpdate + mStaleTime) <= now) - { // stale entry - entry->mLastUpdate = now; - entry->mBalance = 0; - } - else - { - while ((entry->mLastUpdate < now) && (entry->mLastUpdate != 0)) - { - ++entry->mLastUpdate; - entry->mBalance -= (entry->mBalance + mDecaySeconds - 1) / mDecaySeconds; - } - entry->mLastUpdate = now; - } - } - return entry->mBalance != 0; - } - - iBlackList* findEntry(const std::string& source, bool create) - { - iBlackList* ret = nullptr; - - typename BlackListTable::iterator it = mList.find(source); - - if (it != mList.end()) - { - ret = &it->second; - if (!ageEntry(Timer::getElapsedSeconds(), ret) && !create) - { // entry has expired, and we don't need it - mList.erase(it); - ret = nullptr; - } - } - else if (create) - { - ret = &mList[source]; - ret->mLastUpdate = Timer::getElapsedSeconds(); - } - - return ret; - } - -}; - -#endif diff --git a/src/ripple_basics/containers/TaggedCache.cpp b/src/ripple_basics/containers/TaggedCache.cpp index 0d22832f37..1a3953d36b 100644 --- a/src/ripple_basics/containers/TaggedCache.cpp +++ b/src/ripple_basics/containers/TaggedCache.cpp @@ -17,4 +17,138 @@ */ //============================================================================== -SETUP_LOGN (TaggedCacheLog,"TaggedCache") +namespace ripple { + +/* +I guess you can put some items in, make sure they're still there. Let some +time pass, make sure they're gone. Keep a strong pointer to one of them, make +sure you can still find it even after time passes. Create two objects with +the same key, canonicalize them both and make sure you get the same object. +Put an object in but keep a strong pointer to it, advance the clock a lot, +then canonicalize a new object with the same key, make sure you get the +original object. +*/ + +class TaggedCacheTests : public UnitTest +{ +public: + TaggedCacheTests () : UnitTest ( + "TaggedCache", "ripple") + { + } + + void runTest () + { + //Journal const j (journal()); + Journal const j; + + beginTestCase ("Insert"); + + manual_clock clock; + clock.set (0); + + typedef int Key; + typedef std::string Value; + typedef TaggedCacheType Cache; + + Cache c ("test", 1, 1, clock, j); + + // Insert an item, retrieve it, and age it so it gets purged. + { + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 0); + expect (! c.insert (1, "one")); + expect (c.getCacheSize() == 1); + expect (c.getTrackSize() == 1); + + { + std::string s; + expect (c.retrieve (1, s)); + expect (s == "one"); + } + + ++clock; + c.sweep (); + expect (c.getCacheSize () == 0); + expect (c.getTrackSize () == 0); + } + + // Insert an item, maintain a strong pointer, age it, and + // verify that the entry still exists. + { + expect (! c.insert (2, "two")); + expect (c.getCacheSize() == 1); + expect (c.getTrackSize() == 1); + + { + Cache::mapped_ptr p (c.fetch (2)); + expect (p != nullptr); + ++clock; + c.sweep (); + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 1); + } + + // Make sure its gone now that our reference is gone + ++clock; + c.sweep (); + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 0); + } + + // Insert the same key/value pair and make sure we get the same result + { + expect (! c.insert (3, "three")); + + { + Cache::mapped_ptr const p1 (c.fetch (3)); + Cache::mapped_ptr p2 (make_shared ("three")); + c.canonicalize (3, p2); + expect (p1.get() == p2.get()); + } + ++clock; + c.sweep (); + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 0); + } + + // Put an object in but keep a strong pointer to it, advance the clock a lot, + // then canonicalize a new object with the same key, make sure you get the + // original object. + { + // Put an object in + expect (! c.insert (4, "four")); + expect (c.getCacheSize() == 1); + expect (c.getTrackSize() == 1); + + { + // Keep a strong pointer to it + Cache::mapped_ptr p1 (c.fetch (4)); + expect (p1 != nullptr); + expect (c.getCacheSize() == 1); + expect (c.getTrackSize() == 1); + // Advance the clock a lot + ++clock; + c.sweep (); + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 1); + // Canonicalize a new object with the same key + Cache::mapped_ptr p2 (boost::make_shared ("four")); + expect (c.canonicalize (4, p2, false)); + expect (c.getCacheSize() == 1); + expect (c.getTrackSize() == 1); + // Make sure we get the original object + expect (p1.get() == p2.get()); + } + + ++clock; + c.sweep (); + expect (c.getCacheSize() == 0); + expect (c.getTrackSize() == 0); + } + } +}; + +static TaggedCacheTests taggedCacheTests; + +} diff --git a/src/ripple_basics/containers/TaggedCache.h b/src/ripple_basics/containers/TaggedCache.h index 879b576c1f..0a55b47fce 100644 --- a/src/ripple_basics/containers/TaggedCache.h +++ b/src/ripple_basics/containers/TaggedCache.h @@ -17,128 +17,247 @@ */ //============================================================================== -#ifndef RIPPLE_TAGGEDCACHE_H -#define RIPPLE_TAGGEDCACHE_H +#ifndef RIPPLE_TAGGEDCACHE_H_INCLUDED +#define RIPPLE_TAGGEDCACHE_H_INCLUDED -// This class implements a cache and a map. The cache keeps objects alive -// in the map. The map allows multiple code paths that reference objects -// with the same tag to get the same actual object. +#include +#include -// So long as data is in the cache, it will stay in memory. -// If it stays in memory even after it is ejected from the cache, -// the map will track it. - -// CAUTION: Callers must not modify data objects that are stored in the cache -// unless they hold their own lock over all cache operations. +namespace ripple { +// VFALCO NOTE Deprecated struct TaggedCacheLog; -// Common base -class TaggedCache -{ -public: - typedef RippleRecursiveMutex LockType; - typedef LockType::ScopedLockType ScopedLockType; -}; +/** Map/cache combination. + This class implements a cache and a map. The cache keeps objects alive + in the map. The map allows multiple code paths that reference objects + with the same tag to get the same actual object. -/** Combination cache/map container. + So long as data is in the cache, it will stay in memory. + If it stays in memory even after it is ejected from the cache, + the map will track it. - NOTE: - - Timer must have this interface: - - static int Timer::getElapsedSeconds (); + @note Callers must not modify data objects that are stored in the cache + unless they hold their own lock over all cache operations. */ -template -class TaggedCacheType : public TaggedCache +template < + class Key, + class T, + class Hash = std::hash , + class KeyEqual = std::equal_to , + //class Allocator = std::allocator >, + class Mutex = std::recursive_mutex +> +class TaggedCacheType { public: - typedef c_Key key_type; - typedef c_Data data_type; - typedef boost::weak_ptr weak_data_ptr; - typedef boost::shared_ptr data_ptr; + typedef Mutex mutex_type; + // VFALCO DEPRECATED The caller can just use std::unique_lock + typedef std::unique_lock ScopedLockType; + typedef std::lock_guard lock_guard; + typedef Key key_type; + typedef T mapped_type; + // VFALCO TODO Use std::shared_ptr, std::weak_ptr + typedef boost::weak_ptr weak_mapped_ptr; + typedef boost::shared_ptr mapped_ptr; + typedef abstract_clock clock_type; public: - typedef TaggedCache::LockType LockType; - typedef TaggedCache::ScopedLockType ScopedLockType; - - TaggedCacheType (const char* name, int size, int age) - : mLock (static_cast (this), "TaggedCache", __FILE__, __LINE__) - , mName (name) - , mTargetSize (size) - , mTargetAge (age) - , mCacheCount (0) - , mHits (0) - , mMisses (0) + // VFALCO TODO Change expiration_seconds to clock_type::duration + TaggedCacheType (std::string const& name, int size, + clock_type::rep expiration_seconds, clock_type& clock, Journal journal) + : m_journal (journal) + , m_clock (clock) + , m_name (name) + , m_target_size (size) + , m_target_age (std::chrono::seconds (expiration_seconds)) + , m_cache_count (0) + , m_hits (0) + , m_misses (0) { } - int getTargetSize () const; - int getTargetAge () const; - - int getCacheSize (); - int getTrackSize (); - float getHitRate (); - void clearStats (); - - void setTargetSize (int size); - void setTargetAge (int age); - void sweep (); - void clear (); - - /** Refresh the expiration time on a key. - - @param key The key to refresh. - @return `true` if the key was found and the object is cached. - */ - bool refreshIfPresent (const key_type& key) + int getTargetSize () const { - bool found = false; + lock_guard lock (m_mutex); + return m_target_size; + } - // If present, make current in cache - ScopedLockType sl (mLock, __FILE__, __LINE__); + void setTargetSize (int s) + { + lock_guard lock (m_mutex); + m_target_size = s; - cache_iterator cit = mCache.find (key); + if (s > 0) + m_cache.rehash (static_cast ((s + (s >> 2)) / m_cache.max_load_factor () + 1)); - if (cit != mCache.end ()) + if (m_journal.debug) m_journal.debug << + m_name << " target size set to " << s; + } + + clock_type::rep getTargetAge () const + { + lock_guard lock (m_mutex); + return m_target_age.count(); + } + + void setTargetAge (clock_type::rep s) + { + lock_guard lock (m_mutex); + m_target_age = std::chrono::seconds (s); + if (m_journal.debug) m_journal.debug << + m_name << " target age set to " << m_target_age; + } + + int getCacheSize () + { + lock_guard lock (m_mutex); + return m_cache_count; + } + + int getTrackSize () + { + lock_guard lock (m_mutex); + return m_cache.size (); + } + + float getHitRate () + { + lock_guard lock (m_mutex); + return (static_cast (m_hits) * 100) / (1.0f + m_hits + m_misses); + } + + void clearStats () + { + lock_guard lock (m_mutex); + m_hits = 0; + m_misses = 0; + } + + void clear () + { + lock_guard lock (m_mutex); + m_cache.clear (); + m_cache_count = 0; + } + + void sweep () + { + int cacheRemovals = 0; + int mapRemovals = 0; + int cc = 0; + + // Keep references to all the stuff we sweep + // so that we can destroy them outside the lock. + // + std::vector stuffToSweep; + { - cache_entry& entry = cit->second; + lock_guard lock (m_mutex); - if (! entry.isCached ()) + clock_type::time_point const now (m_clock.now()); + clock_type::time_point when_expire; + + if (m_target_size == 0 || + (static_cast (m_cache.size ()) <= m_target_size)) { - // Convert weak to strong. - entry.ptr = entry.lock (); - - if (entry.isCached ()) - { - // We just put the object back in cache - ++mCacheCount; - entry.touch (); - found = true; - } - else - { - // Couldn't get strong pointer, - // object fell out of the cache so remove the entry. - mCache.erase (cit); - } + when_expire = now - m_target_age; } else { - // It's cached so update the timer - entry.touch (); - found = true; + when_expire = now - clock_type::duration ( + m_target_age.count() * m_target_size / m_cache.size ()); + + clock_type::duration const minimumAge ( + std::chrono::seconds (2)); + if (when_expire > (now - minimumAge)) + when_expire = now - minimumAge; + + if (m_journal.trace) m_journal.trace << + m_name << " is growing fast " << m_cache.size () << " of " << m_target_size << + " aging at " << (now - when_expire) << " of " << m_target_age; + } + + stuffToSweep.reserve (m_cache.size ()); + + cache_iterator cit = m_cache.begin (); + + while (cit != m_cache.end ()) + { + if (cit->second.isWeak ()) + { + // weak + if (cit->second.isExpired ()) + { + ++mapRemovals; + cit = m_cache.erase (cit); + } + else + { + ++cit; + } + } + else if (cit->second.last_access <= when_expire) + { + // strong, expired + --m_cache_count; + ++cacheRemovals; + if (cit->second.ptr.unique ()) + { + stuffToSweep.push_back (cit->second.ptr); + ++mapRemovals; + cit = m_cache.erase (cit); + } + else + { + // remains weakly cached + cit->second.ptr.reset (); + ++cit; + } + } + else + { + // strong, not expired + ++cc; + ++cit; + } } } - else - { - // not present - } - return found; + if (m_journal.trace && (mapRemovals || cacheRemovals)) m_journal.trace << + m_name << ": cache = " << m_cache.size () << "-" << cacheRemovals << + ", map-=" << mapRemovals; + + // At this point stuffToSweep will go out of scope outside the lock + // and decrement the reference count on each strong pointer. } - bool del (const key_type& key, bool valid); + bool del (const key_type& key, bool valid) + { + // Remove from cache, if !valid, remove from map too. Returns true if removed from cache + lock_guard lock (m_mutex); + + cache_iterator cit = m_cache.find (key); + + if (cit == m_cache.end ()) + return false; + + Entry& entry = cit->second; + + bool ret = false; + + if (entry.isCached ()) + { + --m_cache_count; + entry.ptr.reset (); + ret = true; + } + + if (!valid || entry.isExpired ()) + m_cache.erase (cit); + + return ret; + } /** Replace aliased objects with originals. @@ -151,368 +270,235 @@ public: @param data A shared pointer to the data corresponding to the object. @param replace `true` if `data` is the up to date version of the object. - @return `true` if the operation was successful. + @return `true` If the key already existed. */ - bool canonicalize (const key_type& key, boost::shared_ptr& data, bool replace = false); - - bool store (const key_type& key, const c_Data& data); - boost::shared_ptr fetch (const key_type& key); - bool retrieve (const key_type& key, c_Data& data); - - LockType& peekMutex () + bool canonicalize (const key_type& key, boost::shared_ptr& data, bool replace = false) { - return mLock; + // Return canonical value, store if needed, refresh in cache + // Return values: true=we had the data already + lock_guard lock (m_mutex); + + cache_iterator cit = m_cache.find (key); + + if (cit == m_cache.end ()) + { + m_cache.insert (cache_pair (key, Entry (m_clock.now(), data))); + ++m_cache_count; + return false; + } + + Entry& entry = cit->second; + entry.touch (m_clock.now()); + + if (entry.isCached ()) + { + if (replace) + { + entry.ptr = data; + entry.weak_ptr = data; + } + else + { + data = entry.ptr; + } + + return true; + } + + mapped_ptr cachedData = entry.lock (); + + if (cachedData) + { + if (replace) + { + entry.ptr = data; + entry.weak_ptr = data; + } + else + { + entry.ptr = cachedData; + data = cachedData; + } + + ++m_cache_count; + return true; + } + + entry.ptr = data; + entry.weak_ptr = data; + ++m_cache_count; + + return false; } -private: - class cache_entry + boost::shared_ptr fetch (const key_type& key) { - public: - int last_use; - data_ptr ptr; - weak_data_ptr weak_ptr; + // fetch us a shared pointer to the stored data object + lock_guard lock (m_mutex); - cache_entry (int l, const data_ptr& d) : last_use (l), ptr (d), weak_ptr (d) + cache_iterator cit = m_cache.find (key); + + if (cit == m_cache.end ()) { - ; + ++m_misses; + return mapped_ptr (); } - bool isWeak () + + Entry& entry = cit->second; + entry.touch (m_clock.now()); + + if (entry.isCached ()) { - return !ptr; + ++m_hits; + return entry.ptr; } - bool isCached () + + entry.ptr = entry.lock (); + + if (entry.isCached ()) { - return !!ptr; + // independent of cache size, so not counted as a hit + ++m_cache_count; + return entry.ptr; } - bool isExpired () - { - return weak_ptr.expired (); - } - data_ptr lock () - { - return weak_ptr.lock (); - } - void touch () - { - last_use = Timer::getElapsedSeconds (); - } - }; - typedef std::pair cache_pair; - typedef boost::unordered_map cache_type; - typedef typename cache_type::iterator cache_iterator; + m_cache.erase (cit); + ++m_misses; + return mapped_ptr (); + } - mutable LockType mLock; + /** Insert the element into the container. + If the key already exists, nothing happens. + @return `true` If the element was inserted + */ + bool insert (key_type const& key, T const& value) + { + mapped_ptr p (boost::make_shared (boost::cref (value))); + return canonicalize (key, p); + } - std::string mName; // Used for logging - int mTargetSize; // Desired number of cache entries (0 = ignore) - int mTargetAge; // Desired maximum cache age - int mCacheCount; // Number of items cached - - cache_type mCache; // Hold strong reference to recent objects - - uint64 mHits, mMisses; -}; - -template -int TaggedCacheType::getTargetSize () const -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - return mTargetSize; -} - -template -void TaggedCacheType::setTargetSize (int s) -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - mTargetSize = s; - - if (s > 0) - mCache.rehash (static_cast ((s + (s >> 2)) / mCache.max_load_factor () + 1)); - - WriteLog (lsDEBUG, TaggedCacheLog) << mName << " target size set to " << s; -} - -template -int TaggedCacheType::getTargetAge () const -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - return mTargetAge; -} - -template -void TaggedCacheType::setTargetAge (int s) -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - mTargetAge = s; - WriteLog (lsDEBUG, TaggedCacheLog) << mName << " target age set to " << s; -} - -template -int TaggedCacheType::getCacheSize () -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - return mCacheCount; -} - -template -int TaggedCacheType::getTrackSize () -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - return mCache.size (); -} - -template -float TaggedCacheType::getHitRate () -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - return (static_cast (mHits) * 100) / (1.0f + mHits + mMisses); -} - -template -void TaggedCacheType::clearStats () -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - mHits = 0; - mMisses = 0; -} - -template -void TaggedCacheType::clear () -{ - ScopedLockType sl (mLock, __FILE__, __LINE__); - mCache.clear (); - mCacheCount = 0; -} - -template -void TaggedCacheType::sweep () -{ - int cacheRemovals = 0; - int mapRemovals = 0; - int cc = 0; - - // Keep references to all the stuff we sweep - // so that we can destroy them outside the lock. + // VFALCO NOTE It looks like this returns a copy of the data in + // the output parameter 'data'. This could be expensive. + // Perhaps it should work like standard containers, which + // simply return an iterator. // - std::vector stuffToSweep; - + bool retrieve (const key_type& key, T& data) { - ScopedLockType sl (mLock, __FILE__, __LINE__); + // retrieve the value of the stored data + mapped_ptr entry = fetch (key); - int const now = Timer::getElapsedSeconds (); - int target = (now < mTargetAge) ? 0 : (now - mTargetAge); + if (!entry) + return false; - if ((mTargetSize != 0) && (static_cast (mCache.size ()) > mTargetSize)) + data = *entry; + return true; + } + + /** Refresh the expiration time on a key. + + @param key The key to refresh. + @return `true` if the key was found and the object is cached. + */ + bool refreshIfPresent (const key_type& key) + { + bool found = false; + + // If present, make current in cache + lock_guard lock (m_mutex); + + cache_iterator cit = m_cache.find (key); + + if (cit != m_cache.end ()) { - target = now - (mTargetAge * mTargetSize / mCache.size ()); + Entry& entry = cit->second; - if ((now > 2) && (target > (now - 2))) - target = now - 2; - - WriteLog (lsINFO, TaggedCacheLog) << mName << " is growing fast " << - mCache.size () << " of " << mTargetSize << - " aging at " << (now - target) << " of " << mTargetAge; - } - - stuffToSweep.reserve (mCache.size ()); - - cache_iterator cit = mCache.begin (); - - while (cit != mCache.end ()) - { - if (cit->second.isWeak ()) + if (! entry.isCached ()) { - // weak - if (cit->second.isExpired ()) + // Convert weak to strong. + entry.ptr = entry.lock (); + + if (entry.isCached ()) { - ++mapRemovals; - cit = mCache.erase (cit); + // We just put the object back in cache + ++m_cache_count; + entry.touch (m_clock.now()); + found = true; } else { - ++cit; - } - } - else if (cit->second.last_use < target) - { - // strong, expired - --mCacheCount; - ++cacheRemovals; - if (cit->second.ptr.unique ()) - { - stuffToSweep.push_back (cit->second.ptr); - ++mapRemovals; - cit = mCache.erase (cit); - } - else - { - // remains weakly cached - cit->second.ptr.reset (); - ++cit; + // Couldn't get strong pointer, + // object fell out of the cache so remove the entry. + m_cache.erase (cit); } } else { - // strong, not expired - ++cc; - ++cit; + // It's cached so update the timer + entry.touch (m_clock.now()); + found = true; } } - } - - if (ShouldLog (lsTRACE, TaggedCacheLog) && (mapRemovals || cacheRemovals)) - { - WriteLog (lsTRACE, TaggedCacheLog) << mName << ": cache = " << mCache.size () << "-" << cacheRemovals << - ", map-=" << mapRemovals; - } - - // At this point stuffToSweep will go out of scope outside the lock - // and decrement the reference count on each strong pointer. -} - -template -bool TaggedCacheType::del (const key_type& key, bool valid) -{ - // Remove from cache, if !valid, remove from map too. Returns true if removed from cache - ScopedLockType sl (mLock, __FILE__, __LINE__); - - cache_iterator cit = mCache.find (key); - - if (cit == mCache.end ()) - return false; - - cache_entry& entry = cit->second; - - bool ret = false; - - if (entry.isCached ()) - { - --mCacheCount; - entry.ptr.reset (); - ret = true; - } - - if (!valid || entry.isExpired ()) - mCache.erase (cit); - - return ret; -} - -// VFALCO NOTE What does it mean to canonicalize the data? -template -bool TaggedCacheType::canonicalize (const key_type& key, boost::shared_ptr& data, bool replace) -{ - // Return canonical value, store if needed, refresh in cache - // Return values: true=we had the data already - ScopedLockType sl (mLock, __FILE__, __LINE__); - - cache_iterator cit = mCache.find (key); - - if (cit == mCache.end ()) - { - mCache.insert (cache_pair (key, cache_entry (Timer::getElapsedSeconds (), data))); - ++mCacheCount; - return false; - } - - cache_entry& entry = cit->second; - entry.touch (); - - if (entry.isCached ()) - { - if (replace) - { - entry.ptr = data; - entry.weak_ptr = data; - } - else - data = entry.ptr; - - return true; - } - - data_ptr cachedData = entry.lock (); - - if (cachedData) - { - if (replace) - { - entry.ptr = data; - entry.weak_ptr = data; - } else { - entry.ptr = cachedData; - data = cachedData; + // not present } - ++mCacheCount; - return true; + return found; } - entry.ptr = data; - entry.weak_ptr = data; - ++mCacheCount; - - return false; -} - -template -boost::shared_ptr TaggedCacheType::fetch (const key_type& key) -{ - // fetch us a shared pointer to the stored data object - ScopedLockType sl (mLock, __FILE__, __LINE__); - - cache_iterator cit = mCache.find (key); - - if (cit == mCache.end ()) + mutex_type& peekMutex () { - ++mMisses; - return data_ptr (); + return m_mutex; } - cache_entry& entry = cit->second; - entry.touch (); - - if (entry.isCached ()) +private: + class Entry { - ++mHits; - return entry.ptr; - } + public: + mapped_ptr ptr; + weak_mapped_ptr weak_ptr; + clock_type::time_point last_access; - entry.ptr = entry.lock (); + Entry (clock_type::time_point const& last_access_, + mapped_ptr const& ptr_) + : ptr (ptr_) + , weak_ptr (ptr_) + , last_access (last_access_) + { + } - if (entry.isCached ()) - { - // independent of cache size, so not counted as a hit - ++mCacheCount; - return entry.ptr; - } + bool isWeak () const { return ptr == nullptr; } + bool isCached () const { return ptr != nullptr; } + bool isExpired () const { return weak_ptr.expired (); } + mapped_ptr lock () { return weak_ptr.lock (); } + void touch (clock_type::time_point const& now) { last_access = now; } + }; - mCache.erase (cit); - ++mMisses; - return data_ptr (); -} + typedef std::pair cache_pair; + typedef std::unordered_map cache_type; + typedef typename cache_type::iterator cache_iterator; -template -bool TaggedCacheType::store (const key_type& key, const c_Data& data) -{ - data_ptr d = boost::make_shared (boost::cref (data)); - return canonicalize (key, d); -} + Journal m_journal; + clock_type& m_clock; -template -bool TaggedCacheType::retrieve (const key_type& key, c_Data& data) -{ - // retrieve the value of the stored data - data_ptr entry = fetch (key); + mutex_type mutable m_mutex; - if (!entry) - return false; + // Used for logging + std::string m_name; + + // Desired number of cache entries (0 = ignore) + int m_target_size; + + // Desired maximum cache age + clock_type::duration m_target_age; + + // Number of items cached + int m_cache_count; + cache_type m_cache; // Hold strong reference to recent objects + uint64 m_hits; + uint64 m_misses; +}; - data = *entry; - return true; } #endif diff --git a/src/ripple_basics/ripple_basics.h b/src/ripple_basics/ripple_basics.h index ba20b96bcd..ec947130cd 100644 --- a/src/ripple_basics/ripple_basics.h +++ b/src/ripple_basics/ripple_basics.h @@ -30,7 +30,7 @@ #include -#ifndef RIPPLE_TRACK_MUTEXES +#ifndef RIPPLE_TRACK_MUTEXES # define RIPPLE_TRACK_MUTEXES 0 #endif @@ -58,8 +58,7 @@ namespace boost #include "../ripple/types/ripple_types.h" -namespace ripple -{ +namespace ripple { using namespace beast; @@ -83,10 +82,10 @@ using namespace beast; #include "containers/KeyCache.h" #include "containers/RangeSet.h" -#include "containers/BlackList.h" -#include "containers/TaggedCache.h" #include "containers/SyncUnorderedMap.h" } +#include "containers/TaggedCache.h" + #endif diff --git a/src/ripple_core/nodestore/impl/DatabaseImp.h b/src/ripple_core/nodestore/impl/DatabaseImp.h index 4ca6df9141..cc17ebc32d 100644 --- a/src/ripple_core/nodestore/impl/DatabaseImp.h +++ b/src/ripple_core/nodestore/impl/DatabaseImp.h @@ -17,8 +17,7 @@ */ //============================================================================== -namespace NodeStore -{ +namespace NodeStore { class DatabaseImp : public Database @@ -35,7 +34,9 @@ public: , m_backend (createBackend (backendParameters, scheduler, journal)) , m_fastBackend ((fastBackendParameters.size () > 0) ? createBackend (fastBackendParameters, scheduler, journal) : nullptr) - , m_cache ("NodeStore", 16384, 300) + , m_cache ("NodeStore", 16384, 300, + get_abstract_clock (), + LogPartition::getJournal ()) { } @@ -304,7 +305,7 @@ private: ScopedPointer m_fastBackend; // VFALCO NOTE What are these things for? We need comments. - TaggedCacheType m_cache; + TaggedCacheType m_cache; }; //------------------------------------------------------------------------------