ARCHIVE/rippled
1
0
Fork 0
mirror of https://github.com/XRPLF/rippled.git synced 2025-11-28 23:15:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor KeyCache

Browse Source
This commit is contained in:
Vinnie Falco
2014-01-13 14:18:33 -08:00
parent cac1d555be
commit 68501763dd
11 changed files with 259 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Builds/VisualStudio2012/RippleD.vcxproj
View File

@@ -1394,6 +1394,12 @@
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\src\ripple_basics\containers\KeyCache.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|x64'">true</ExcludedFromBuild>
</ClCompile>
<ClCompile Include="..\..\src\ripple_basics\containers\RangeSet.cpp">
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</ExcludedFromBuild>
<ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</ExcludedFromBuild>

3
Builds/VisualStudio2012/RippleD.vcxproj.filters
View File

@@ -1416,6 +1416,9 @@
<ClCompile Include="..\..\src\ripple_core\nodestore\backend\RocksDBFactory.cpp">
<Filter>[2] Old Ripple\ripple_core\nodestore\backend</Filter>
</ClCompile>
<ClCompile Include="..\..\src\ripple_basics\containers\KeyCache.cpp">
<Filter>[2] Old Ripple\ripple_basics\containers</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\src\ripple_basics\containers\KeyCache.h">

9
src/ripple_app/ledger/InboundLedgers.cpp
View File

@@ -31,8 +31,9 @@ public:
explicit InboundLedgersImp (Stoppable& parent)
: Stoppable ("InboundLedgers", parent)
, mLock (this, "InboundLedger", __FILE__, __LINE__)
, mRecentFailures ("LedgerAcquireRecentFailures", 0,
kReacquireIntervalSeconds)
, mRecentFailures ("LedgerAcquireRecentFailures",
get_abstract_clock <std::chrono::steady_clock, std::chrono::seconds> (),
0, kReacquireIntervalSeconds)
{
}
@@ -225,12 +226,12 @@ public:
void logFailure (uint256 const& h)
{
mRecentFailures.add (h);
mRecentFailures.insert (h);
}
bool isFailure (uint256 const& h)
{
return mRecentFailures.isPresent (h, false);
return mRecentFailures.exists (h);
}
void doLedgerData (Job&, LedgerHash hash)

5
src/ripple_app/shamap/SHAMap.h
View File

@@ -238,13 +238,16 @@ public:
std::list<fetchPackEntry_t> getFetchPack (SHAMap * have, bool includeLeaves, int max);
void getFetchPack (SHAMap * have, bool includeLeaves, int max, std::function<void (const uint256&, const Blob&)>);
// VFALCO NOTE These static members should be moved into a
// new Application singleton class.
//
// tree node cache operations
static SHAMapTreeNode::pointer getCache (uint256 const& hash, SHAMapNode const& id);
static void canonicalize (uint256 const& hash, SHAMapTreeNode::pointer&);
static int getFullBelowSize ()
{
return fullBelowCache.getSize ();
return fullBelowCache.size ();
}
static int getTreeNodeSize ()
{

11
src/ripple_app/shamap/SHAMapSync.cpp
View File

@@ -21,7 +21,10 @@
static const uint256 uZero;
KeyCache <uint256, UptimeTimerAdapter> SHAMap::fullBelowCache ("fullBelowCache", 524288, 240);
KeyCache <uint256, UptimeTimerAdapter> SHAMap::fullBelowCache (
"fullBelowCache",
get_abstract_clock <std::chrono::steady_clock, std::chrono::seconds> (),
524288, 240);
void SHAMap::visitLeaves (std::function<void (SHAMapItem::ref item)> function)
{
@@ -149,7 +152,7 @@ void SHAMap::getMissingNodes (std::vector<SHAMapNode>& nodeIDs, std::vector<uint
{
uint256 const& childHash = node->getChildHash (branch);
if (!fullBelowCache.isPresent (childHash))
if (! fullBelowCache.touch_if_exists (childHash))
{
SHAMapNode childID = node->getChildNodeID (branch);
SHAMapTreeNode* d = getNodePointerNT (childID, childHash, filter);
@@ -184,7 +187,7 @@ void SHAMap::getMissingNodes (std::vector<SHAMapNode>& nodeIDs, std::vector<uint
{ // No partial node encountered below this node
node->setFullBelow ();
if (mType == smtSTATE)
fullBelowCache.add (node->getNodeHash ());
fullBelowCache.insert (node->getNodeHash ());
}
if (stack.empty ())
@@ -394,7 +397,7 @@ SHAMapAddNode SHAMap::addKnownNode (const SHAMapNode& node, Blob const& rawNode,
return SHAMapAddNode::invalid ();
}
if (fullBelowCache.isPresent (iNode->getChildHash (branch)))
if (fullBelowCache.touch_if_exists (iNode->getChildHash (branch)))
return SHAMapAddNode::duplicate ();
SHAMapTreeNode *nextNode = getNodePointerNT (iNode->getChildNodeID (branch), iNode->getChildHash (branch), filter);

100
src/ripple_basics/containers/KeyCache.cpp Normal file
View File

@@ -0,0 +1,100 @@
//------------------------------------------------------------------------------
/*
This file is part of rippled: https://github.com/ripple/rippled
Copyright (c) 2012, 2013 Ripple Labs Inc.
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "KeyCache.h"
namespace ripple {
class KeyCacheTests : public UnitTest
{
public:
void runTest ()
{
beginTestCase ("Insert");
manual_clock <std::chrono::seconds> clock;
clock.set (0);
typedef std::string Key;
typedef KeyCache <Key> Cache;
// Insert an item, retrieve it, and age it so it gets purged.
{
Cache c ("test", clock, 1, 2);
expect (c.size () == 0);
expect (c.insert ("one"));
expect (! c.insert ("one"));
expect (c.size () == 1);
expect (c.exists ("one"));
expect (c.touch_if_exists ("one"));
++clock;
c.sweep ();
expect (c.size () == 1);
expect (c.exists ("one"));
++clock;
c.sweep ();
expect (c.size () == 0);
expect (! c.exists ("one"));
expect (! c.touch_if_exists ("one"));
}
// Insert two items, have one expire
{
Cache c ("test", clock, 2, 2);
expect (c.insert ("one"));
expect (c.size () == 1);
expect (c.insert ("two"));
expect (c.size () == 2);
++clock;
c.sweep ();
expect (c.size () == 2);
expect (c.touch_if_exists ("two"));
++clock;
c.sweep ();
expect (c.size () == 1);
expect (c.exists ("two"));
}
// Insert three items (1 over limit), sweep
{
Cache c ("test", clock, 2, 3);
expect (c.insert ("one"));
++clock;
expect (c.insert ("two"));
++clock;
expect (c.insert ("three"));
++clock;
expect (c.size () == 3);
c.sweep ();
expect (c.size () < 3);
}
}
KeyCacheTests () : UnitTest (
"KeyCache", "ripple")
{
}
};
static KeyCacheTests keyCacheTests;
}

260
src/ripple_basics/containers/KeyCache.h
View File

@@ -20,189 +20,186 @@
#ifndef RIPPLE_KEYCACHE_H_INCLUDED
#define RIPPLE_KEYCACHE_H_INCLUDED
// This tag is for helping track the locks
struct KeyCacheBase { };
#include "beast/beast/chrono/abstract_clock.h"
#include <mutex>
#include <unordered_map>
namespace ripple {
/** Maintains a cache of keys with no associated data.
The cache has a target size and an expiration time. When cached items become
older than the maximum age they are eligible for removal during a
call to @ref sweep.
@note
Timer must provide this function:
@code
static int getElapsedSeconds ();
@endcode
*/
template <class Key, class Timer>
class KeyCache : public KeyCacheBase
// VFALCO TODO Figure out how to pass through the allocator
template <
class Key,
class Hash = std::hash <Key>,
class KeyEqual = std::equal_to <Key>,
//class Allocator = std::allocator <std::pair <Key const, Entry>>,
class Mutex = std::mutex
>
class KeyCache
{
public:
/** Provides a type for the key.
*/
typedef Key key_type;
typedef abstract_clock <std::chrono::seconds> clock_type;
private:
struct Entry
{
explicit Entry (clock_type::time_point const& last_access_)
: last_access (last_access_)
{
}
clock_type::time_point last_access;
};
typedef std::unordered_map <key_type, Entry> map_type;
typedef typename map_type::iterator iterator;
typedef std::lock_guard <Mutex> lock_guard;
Mutex mutable m_mutex;
map_type m_map;
clock_type& m_clock;
std::string const m_name;
unsigned int m_target_size;
clock_type::duration m_target_age;
public:
typedef typename map_type::size_type size_type;
/** Construct with the specified name.
@param size The initial target size.
@param age The initial expiration time.
*/
KeyCache (const std::string& name,
int size = 0,
int age = 120)
: mLock (static_cast <KeyCacheBase*> (this), String ("KeyCache") +
"('" + name + "')", __FILE__, __LINE__)
, mName (name)
, mTargetSize (size)
, mTargetAge (age)
KeyCache (std::string const& name,
clock_type& clock, size_type target_size = 0,
clock_type::rep expiration_seconds = 120)
: m_clock (clock)
, m_name (name)
, m_target_size (target_size)
, m_target_age (std::chrono::seconds (expiration_seconds))
{
assert ((size >= 0) && (age > 2));
assert (m_target_size >= 0);
}
/** Returns the current size.
*/
unsigned int getSize ()
//--------------------------------------------------------------------------
/** Retrieve the name of this object. */
std::string const& name () const
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
return mCache.size ();
return m_name;
}
/** Returns the desired target size.
*/
unsigned int getTargetSize ()
/** Returns the number of items in the container. */
size_type size () const
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
return mTargetSize;
lock_guard lock (m_mutex);
return m_map.size ();
}
/** Returns the desired target age.
*/
unsigned int getTargetAge ()
/** Empty the cache */
void clear ()
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
return mTargetAge;
lock_guard lock (m_mutex);
m_map.clear ();
}
/** Simultaneously set the target size and age.
@param size The target size.
@param age The target age.
/** Returns `true` if the key was found.
Does not update the last access time.
*/
void setTargets (int size, int age)
template <class KeyComparable>
bool exists (KeyComparable const& key) const
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
mTargetSize = size;
mTargetAge = age;
assert ((mTargetSize >= 0) && (mTargetAge > 2));
lock_guard lock (m_mutex);
typename map_type::const_iterator const iter (m_map.find (key));
return iter != m_map.end ();
}
/** Retrieve the name of this object.
/** Insert the specified key.
The last access time is refreshed in all cases.
@return `true` If the key was newly inserted.
*/
std::string const& getName ()
bool insert (Key const& key)
{
return mName;
}
/** Determine if the specified key is cached, and optionally refresh it.
@param key The key to check
@param refresh Whether or not to refresh the entry.
@return `true` if the key was found.
*/
bool isPresent (const key_type& key, bool refresh = true)
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
map_iterator it = mCache.find (key);
if (it == mCache.end ())
lock_guard lock (m_mutex);
clock_type::time_point const now (m_clock.now ());
std::pair <iterator, bool> result (m_map.emplace (
std::piecewise_construct, std::make_tuple (key),
std::make_tuple (now)));
if (! result.second)
{
result.first->second.last_access = now;
return false;
}
return true;
}
if (refresh)
it->second = Timer::getElapsedSeconds ();
/** Refresh the last access time on a key if present.
@return `true` If the key was found.
*/
template <class KeyComparable>
bool touch_if_exists (KeyComparable const& key)
{
lock_guard lock (m_mutex);
iterator const iter (m_map.find (key));
if (iter == m_map.end ())
return false;
iter->second.last_access = m_clock.now ();
return true;
}
/** Remove the specified cache entry.
@param key The key to remove.
@return `false` if the key was not found.
@return `false` If the key was not found.
*/
bool del (const key_type& key)
bool erase (key_type const& key)
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
map_iterator it = mCache.find (key);
if (it == mCache.end ())
return false;
mCache.erase (it);
return true;
lock_guard lock (m_mutex);
return m_map.erase (key) > 0;
}
/** Add the specified cache entry.
@param key The key to add.
@return `true` if the key did not previously exist.
*/
bool add (const key_type& key)
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
map_iterator it = mCache.find (key);
if (it != mCache.end ())
{
it->second = Timer::getElapsedSeconds ();
return false;
}
mCache.insert (std::make_pair (key, Timer::getElapsedSeconds ()));
return true;
}
/** Empty the cache
*/
void clear ()
{
ScopedLockType sl (mLock, __FILE__, __LINE__);
mCache.clear ();
}
/** Remove stale entries from the cache.
*/
/** Remove stale entries from the cache. */
void sweep ()
{
int now = Timer::getElapsedSeconds ();
ScopedLockType sl (mLock, __FILE__, __LINE__);
clock_type::time_point const now (m_clock.now ());
clock_type::time_point when_expire;
int target;
lock_guard lock (m_mutex);
if ((mTargetSize == 0) || (mCache.size () <= mTargetSize))
target = now - mTargetAge;
if (m_target_size == 0 ||
(m_map.size () <= m_target_size))
{
when_expire = now - m_target_age;
}
else
{
target = now - (mTargetAge * mTargetSize / mCache.size ());
when_expire = now - clock_type::duration (
m_target_age.count() * m_target_size / m_map.size ());
if (target > (now - 2))
target = now - 2;
clock_type::duration const minimumAge (
std::chrono::seconds (1));
if (when_expire > (now - minimumAge))
when_expire = now - minimumAge;
}
map_iterator it = mCache.begin ();
iterator it = m_map.begin ();
while (it != mCache.end ())
while (it != m_map.end ())
{
if (it->second > now)
if (it->second.last_access > now)
{
it->second = now;
it->second.last_access = now;
++it;
}
else if (it->second < target)
else if (it->second.last_access <= when_expire)
{
it = mCache.erase (it);
it = m_map.erase (it);
}
else
{
@@ -210,21 +207,8 @@ public:
}
}
}
protected:
/** Provides a type for the underlying map. */
typedef boost::unordered_map<key_type, int> map_type;
/** The type of the iterator used for traversals. */
typedef typename map_type::iterator map_iterator;
typedef RippleMutex LockType;
typedef LockType::ScopedLockType ScopedLockType;
LockType mLock;
std::string const mName;
map_type mCache;
unsigned int mTargetSize, mTargetAge;
};
}
#endif

10
src/ripple_basics/containers/TaggedCache.cpp
View File

@@ -32,11 +32,6 @@ original object.
class TaggedCacheTests : public UnitTest
{
public:
TaggedCacheTests () : UnitTest (
"TaggedCache", "ripple")
{
}
void runTest ()
{
//Journal const j (journal());
@@ -147,6 +142,11 @@ public:
expect (c.getTrackSize() == 0);
}
}
TaggedCacheTests () : UnitTest (
"TaggedCache", "ripple")
{
}
};
static TaggedCacheTests taggedCacheTests;

9
src/ripple_basics/containers/TaggedCache.h
View File

@@ -40,12 +40,13 @@ struct TaggedCacheLog;
@note Callers must not modify data objects that are stored in the cache
unless they hold their own lock over all cache operations.
*/
// VFALCO TODO Figure out how to pass through the allocator
template <
class Key,
class T,
class Hash = std::hash <Key>,
class KeyEqual = std::equal_to <Key>,
//class Allocator = std::allocator <std::pair <Key const, T>>,
//class Allocator = std::allocator <std::pair <Key const, Entry>>,
class Mutex = std::recursive_mutex
>
class TaggedCacheType
@@ -153,11 +154,11 @@ public:
std::vector <mapped_ptr> stuffToSweep;
{
lock_guard lock (m_mutex);
clock_type::time_point const now (m_clock.now());
clock_type::time_point when_expire;
lock_guard lock (m_mutex);
if (m_target_size == 0 ||
(static_cast<int> (m_cache.size ()) <= m_target_size))
{
@@ -169,7 +170,7 @@ public:
m_target_age.count() * m_target_size / m_cache.size ());
clock_type::duration const minimumAge (
std::chrono::seconds (2));
std::chrono::seconds (1));
if (when_expire > (now - minimumAge))
when_expire = now - minimumAge;

1
src/ripple_basics/ripple_basics.cpp
View File

@@ -45,6 +45,7 @@
namespace ripple {
#include "containers/KeyCache.cpp"
#include "containers/RangeSet.cpp"
#include "containers/TaggedCache.cpp"

2
src/ripple_basics/ripple_basics.h
View File

@@ -80,12 +80,12 @@ using namespace beast;
#include "utility/Time.h"
#include "utility/UptimeTimer.h"
#include "containers/KeyCache.h"
#include "containers/RangeSet.h"
#include "containers/SyncUnorderedMap.h"
}
#include "containers/KeyCache.h"
#include "containers/TaggedCache.h"
#endif