From 19018e895905adfe70030f6c03e7ec8d03f81aef Mon Sep 17 00:00:00 2001 From: Mark Travis Date: Fri, 17 Sep 2021 15:48:33 -0700 Subject: [PATCH] Introduce partitioned unordered maps: This commit implements partitioned unordered maps and makes it possible to traverse such a map in parallel, allowing for more efficient use of CPU resources. The `CachedSLEs`, `TaggedCache`, and `KeyCache` classes make use of the new functionality, which should improve performance. --- Builds/CMake/RippledCore.cmake | 2 +- src/ripple/app/ledger/impl/InboundLedgers.cpp | 22 +- src/ripple/app/ledger/impl/LedgerReplayer.cpp | 66 +-- src/ripple/app/main/Application.cpp | 13 +- src/ripple/app/main/Application.h | 14 +- src/ripple/basics/KeyCache.h | 310 +----------- src/ripple/basics/SHAMapHash.h | 113 +++++ src/ripple/basics/TaggedCache.h | 455 ++++++++++++------ src/ripple/basics/UnorderedContainers.h | 10 + .../basics/impl/partitioned_unordered_map.cpp | 78 +++ src/ripple/basics/partitioned_unordered_map.h | 409 ++++++++++++++++ src/ripple/consensus/Consensus.h | 3 + src/ripple/consensus/Validations.h | 71 +-- src/ripple/ledger/CachedSLEs.h | 88 +--- src/ripple/ledger/impl/CachedSLEs.cpp | 57 --- src/ripple/nodestore/Database.h | 1 - src/ripple/nodestore/impl/Shard.h | 3 +- src/ripple/shamap/FullBelowCache.h | 17 +- src/ripple/shamap/SHAMapTreeNode.h | 83 +--- src/ripple/shamap/impl/NodeFamily.cpp | 1 + src/ripple/shamap/impl/ShardFamily.cpp | 1 + src/test/basics/KeyCache_test.cpp | 19 +- src/test/basics/TaggedCache_test.cpp | 3 +- src/test/consensus/Validations_test.cpp | 15 +- src/test/csf/Peer.h | 2 +- src/test/shamap/common.h | 3 +- 26 files changed, 1089 insertions(+), 770 deletions(-) create mode 100644 src/ripple/basics/SHAMapHash.h create mode 100644 src/ripple/basics/impl/partitioned_unordered_map.cpp create mode 100644 src/ripple/basics/partitioned_unordered_map.h delete mode 100644 src/ripple/ledger/impl/CachedSLEs.cpp diff --git a/Builds/CMake/RippledCore.cmake b/Builds/CMake/RippledCore.cmake index 731ea8bf6..e8fc5327d 100644 --- a/Builds/CMake/RippledCore.cmake +++ b/Builds/CMake/RippledCore.cmake @@ -447,6 +447,7 @@ target_sources (rippled PRIVATE src/ripple/basics/impl/UptimeClock.cpp src/ripple/basics/impl/make_SSLContext.cpp src/ripple/basics/impl/mulDiv.cpp + src/ripple/basics/impl/partitioned_unordered_map.cpp #[===============================[ main sources: subdir: conditions @@ -483,7 +484,6 @@ target_sources (rippled PRIVATE src/ripple/ledger/impl/ApplyViewBase.cpp src/ripple/ledger/impl/ApplyViewImpl.cpp src/ripple/ledger/impl/BookDirs.cpp - src/ripple/ledger/impl/CachedSLEs.cpp src/ripple/ledger/impl/CachedView.cpp src/ripple/ledger/impl/Directory.cpp src/ripple/ledger/impl/OpenView.cpp diff --git a/src/ripple/app/ledger/impl/InboundLedgers.cpp b/src/ripple/app/ledger/impl/InboundLedgers.cpp index 2f8a07138..8ee3443a2 100644 --- a/src/ripple/app/ledger/impl/InboundLedgers.cpp +++ b/src/ripple/app/ledger/impl/InboundLedgers.cpp @@ -30,6 +30,7 @@ #include #include #include +#include namespace ripple { @@ -347,27 +348,29 @@ public: void sweep() override { - clock_type::time_point const now(m_clock.now()); + auto const start = m_clock.now(); // Make a list of things to sweep, while holding the lock std::vector stuffToSweep; std::size_t total; + { ScopedLockType sl(mLock); MapType::iterator it(mLedgers.begin()); total = mLedgers.size(); + stuffToSweep.reserve(total); while (it != mLedgers.end()) { - if (it->second->getLastAction() > now) + auto const la = it->second->getLastAction(); + + if (la > start) { it->second->touch(); ++it; } - else if ( - (it->second->getLastAction() + std::chrono::minutes(1)) < - now) + else if ((la + std::chrono::minutes(1)) < start) { stuffToSweep.push_back(it->second); // shouldn't cause the actual final delete @@ -383,8 +386,13 @@ public: beast::expire(mRecentFailures, kReacquireInterval); } - JLOG(j_.debug()) << "Swept " << stuffToSweep.size() << " out of " - << total << " inbound ledgers."; + JLOG(j_.debug()) + << "Swept " << stuffToSweep.size() << " out of " << total + << " inbound ledgers. Duration: " + << std::chrono::duration_cast( + m_clock.now() - start) + .count() + << "ms"; } void diff --git a/src/ripple/app/ledger/impl/LedgerReplayer.cpp b/src/ripple/app/ledger/impl/LedgerReplayer.cpp index 9ec5c6f2c..c7aa5d9ca 100644 --- a/src/ripple/app/ledger/impl/LedgerReplayer.cpp +++ b/src/ripple/app/ledger/impl/LedgerReplayer.cpp @@ -218,39 +218,47 @@ LedgerReplayer::gotReplayDelta( void LedgerReplayer::sweep() { - std::lock_guard lock(mtx_); - JLOG(j_.debug()) << "Sweeping, LedgerReplayer has " << tasks_.size() - << " tasks, " << skipLists_.size() << " skipLists, and " - << deltas_.size() << " deltas."; + auto const start = std::chrono::steady_clock::now(); + { + std::lock_guard lock(mtx_); + JLOG(j_.debug()) << "Sweeping, LedgerReplayer has " << tasks_.size() + << " tasks, " << skipLists_.size() + << " skipLists, and " << deltas_.size() << " deltas."; - tasks_.erase( - std::remove_if( - tasks_.begin(), - tasks_.end(), - [this](auto const& t) -> bool { - if (t->finished()) - { - JLOG(j_.debug()) - << "Sweep task " << t->getTaskParameter().finishHash_; - return true; - } - return false; - }), - tasks_.end()); + tasks_.erase( + std::remove_if( + tasks_.begin(), + tasks_.end(), + [this](auto const& t) -> bool { + if (t->finished()) + { + JLOG(j_.debug()) << "Sweep task " + << t->getTaskParameter().finishHash_; + return true; + } + return false; + }), + tasks_.end()); - auto removeCannotLocked = [](auto& subTasks) { - for (auto it = subTasks.begin(); it != subTasks.end();) - { - if (auto item = it->second.lock(); !item) + auto removeCannotLocked = [](auto& subTasks) { + for (auto it = subTasks.begin(); it != subTasks.end();) { - it = subTasks.erase(it); + if (auto item = it->second.lock(); !item) + { + it = subTasks.erase(it); + } + else + ++it; } - else - ++it; - } - }; - removeCannotLocked(skipLists_); - removeCannotLocked(deltas_); + }; + removeCannotLocked(skipLists_); + removeCannotLocked(deltas_); + } + JLOG(j_.debug()) << " LedgerReplayer sweep lock duration " + << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() + << "ms"; } void diff --git a/src/ripple/app/main/Application.cpp b/src/ripple/app/main/Application.cpp index 6dad8f240..29451bfc3 100644 --- a/src/ripple/app/main/Application.cpp +++ b/src/ripple/app/main/Application.cpp @@ -79,12 +79,14 @@ #include +#include #include #include #include #include #include #include +#include #include #include @@ -327,7 +329,12 @@ public: stopwatch(), logs_->journal("TaggedCache")) - , cachedSLEs_(std::chrono::minutes(1), stopwatch()) + , cachedSLEs_( + "Cached SLEs", + 0, + std::chrono::minutes(1), + stopwatch(), + logs_->journal("CachedSLEs")) , validatorKeys_(*config_, m_journal) @@ -1146,11 +1153,11 @@ public: shardStore_->sweep(); getLedgerMaster().sweep(); getTempNodeCache().sweep(); - getValidations().expire(); + getValidations().expire(m_journal); getInboundLedgers().sweep(); getLedgerReplayer().sweep(); m_acceptedLedgerCache.sweep(); - cachedSLEs_.expire(); + cachedSLEs_.sweep(); #ifdef RIPPLED_REPORTING if (auto pg = dynamic_cast( diff --git a/src/ripple/app/main/Application.h b/src/ripple/app/main/Application.h index 3c31e10bb..0fc927ff6 100644 --- a/src/ripple/app/main/Application.h +++ b/src/ripple/app/main/Application.h @@ -52,7 +52,19 @@ class ShardArchiveHandler; // VFALCO TODO Fix forward declares required for header dependency loops class AmendmentTable; -class CachedSLEs; + +template < + class Key, + class T, + bool IsKeyCache, + class Hash, + class KeyEqual, + class Mutex> +class TaggedCache; +class STLedgerEntry; +using SLE = STLedgerEntry; +using CachedSLEs = TaggedCache; + class CollectorManager; class Family; class HashRouter; diff --git a/src/ripple/basics/KeyCache.h b/src/ripple/basics/KeyCache.h index 9f6266310..d8fa4910a 100644 --- a/src/ripple/basics/KeyCache.h +++ b/src/ripple/basics/KeyCache.h @@ -1,7 +1,7 @@ //------------------------------------------------------------------------------ /* This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. + Copyright (c) 2021 Ripple Labs Inc. Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above @@ -17,312 +17,16 @@ */ //============================================================================== -#ifndef RIPPLE_BASICS_KEYCACHE_H_INCLUDED -#define RIPPLE_BASICS_KEYCACHE_H_INCLUDED +#ifndef RIPPLE_BASICS_KEYCACHE_H +#define RIPPLE_BASICS_KEYCACHE_H -#include -#include -#include -#include -#include +#include +#include namespace ripple { -/** Maintains a cache of keys with no associated data. - - The cache has a target size and an expiration time. When cached items become - older than the maximum age they are eligible for removal during a - call to @ref sweep. -*/ -template < - class Key, - class Hash = hardened_hash<>, - class KeyEqual = std::equal_to, - // class Allocator = std::allocator >, - class Mutex = std::mutex> -class KeyCache -{ -public: - using key_type = Key; - using clock_type = beast::abstract_clock; - -private: - struct Stats - { - template - Stats( - std::string const& prefix, - Handler const& handler, - beast::insight::Collector::ptr const& collector) - : hook(collector->make_hook(handler)) - , size(collector->make_gauge(prefix, "size")) - , hit_rate(collector->make_gauge(prefix, "hit_rate")) - , hits(0) - , misses(0) - { - } - - beast::insight::Hook hook; - beast::insight::Gauge size; - beast::insight::Gauge hit_rate; - - std::size_t hits; - std::size_t misses; - }; - - struct Entry - { - explicit Entry(clock_type::time_point const& last_access_) - : last_access(last_access_) - { - } - - clock_type::time_point last_access; - }; - - using map_type = hardened_hash_map; - using iterator = typename map_type::iterator; - -public: - using size_type = typename map_type::size_type; - -private: - Mutex mutable m_mutex; - map_type m_map; - Stats mutable m_stats; - clock_type& m_clock; - std::string const m_name; - size_type m_target_size; - clock_type::duration m_target_age; - -public: - /** Construct with the specified name. - - @param size The initial target size. - @param age The initial expiration time. - */ - KeyCache( - std::string const& name, - clock_type& clock, - beast::insight::Collector::ptr const& collector, - size_type target_size = 0, - std::chrono::seconds expiration = std::chrono::minutes{2}) - : m_stats(name, std::bind(&KeyCache::collect_metrics, this), collector) - , m_clock(clock) - , m_name(name) - , m_target_size(target_size) - , m_target_age(expiration) - { - } - - // VFALCO TODO Use a forwarding constructor call here - KeyCache( - std::string const& name, - clock_type& clock, - size_type target_size = 0, - std::chrono::seconds expiration = std::chrono::minutes{2}) - : m_stats( - name, - std::bind(&KeyCache::collect_metrics, this), - beast::insight::NullCollector::New()) - , m_clock(clock) - , m_name(name) - , m_target_size(target_size) - , m_target_age(expiration) - { - } - - //-------------------------------------------------------------------------- - - /** Retrieve the name of this object. */ - std::string const& - name() const - { - return m_name; - } - - /** Return the clock associated with the cache. */ - clock_type& - clock() - { - return m_clock; - } - - /** Returns the number of items in the container. */ - size_type - size() const - { - std::lock_guard lock(m_mutex); - return m_map.size(); - } - - /** Empty the cache */ - void - clear() - { - std::lock_guard lock(m_mutex); - m_map.clear(); - } - - void - reset() - { - std::lock_guard lock(m_mutex); - m_map.clear(); - m_stats.hits = 0; - m_stats.misses = 0; - } - - void - setTargetSize(size_type s) - { - std::lock_guard lock(m_mutex); - m_target_size = s; - } - - void - setTargetAge(std::chrono::seconds s) - { - std::lock_guard lock(m_mutex); - m_target_age = s; - } - - /** Returns `true` if the key was found. - Does not update the last access time. - */ - template - bool - exists(KeyComparable const& key) const - { - std::lock_guard lock(m_mutex); - typename map_type::const_iterator const iter(m_map.find(key)); - if (iter != m_map.end()) - { - ++m_stats.hits; - return true; - } - ++m_stats.misses; - return false; - } - - /** Insert the specified key. - The last access time is refreshed in all cases. - @return `true` If the key was newly inserted. - */ - bool - insert(Key const& key) - { - std::lock_guard lock(m_mutex); - clock_type::time_point const now(m_clock.now()); - auto [it, inserted] = m_map.emplace( - std::piecewise_construct, - std::forward_as_tuple(key), - std::forward_as_tuple(now)); - if (!inserted) - { - it->second.last_access = now; - return false; - } - return true; - } - - /** Refresh the last access time on a key if present. - @return `true` If the key was found. - */ - template - bool - touch_if_exists(KeyComparable const& key) - { - std::lock_guard lock(m_mutex); - iterator const iter(m_map.find(key)); - if (iter == m_map.end()) - { - ++m_stats.misses; - return false; - } - iter->second.last_access = m_clock.now(); - ++m_stats.hits; - return true; - } - - /** Remove the specified cache entry. - @param key The key to remove. - @return `false` If the key was not found. - */ - bool - erase(key_type const& key) - { - std::lock_guard lock(m_mutex); - if (m_map.erase(key) > 0) - { - ++m_stats.hits; - return true; - } - ++m_stats.misses; - return false; - } - - /** Remove stale entries from the cache. */ - void - sweep() - { - clock_type::time_point const now(m_clock.now()); - clock_type::time_point when_expire; - - std::lock_guard lock(m_mutex); - - if (m_target_size == 0 || (m_map.size() <= m_target_size)) - { - when_expire = now - m_target_age; - } - else - { - when_expire = now - m_target_age * m_target_size / m_map.size(); - - clock_type::duration const minimumAge(std::chrono::seconds(1)); - if (when_expire > (now - minimumAge)) - when_expire = now - minimumAge; - } - - iterator it = m_map.begin(); - - while (it != m_map.end()) - { - if (it->second.last_access > now) - { - it->second.last_access = now; - ++it; - } - else if (it->second.last_access <= when_expire) - { - it = m_map.erase(it); - } - else - { - ++it; - } - } - } - -private: - void - collect_metrics() - { - m_stats.size.set(size()); - - { - beast::insight::Gauge::value_type hit_rate(0); - { - std::lock_guard lock(m_mutex); - auto const total(m_stats.hits + m_stats.misses); - if (total != 0) - hit_rate = (m_stats.hits * 100) / total; - } - m_stats.hit_rate.set(hit_rate); - } - } -}; +using KeyCache = TaggedCache; } // namespace ripple -#endif +#endif // RIPPLE_BASICS_KEYCACHE_H diff --git a/src/ripple/basics/SHAMapHash.h b/src/ripple/basics/SHAMapHash.h new file mode 100644 index 000000000..796510ba1 --- /dev/null +++ b/src/ripple/basics/SHAMapHash.h @@ -0,0 +1,113 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED +#define RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED + +#include + +#include + +namespace ripple { + +// A SHAMapHash is the hash of a node in a SHAMap, and also the +// type of the hash of the entire SHAMap. + +class SHAMapHash +{ + uint256 hash_; + +public: + SHAMapHash() = default; + explicit SHAMapHash(uint256 const& hash) : hash_(hash) + { + } + + uint256 const& + as_uint256() const + { + return hash_; + } + uint256& + as_uint256() + { + return hash_; + } + bool + isZero() const + { + return hash_.isZero(); + } + bool + isNonZero() const + { + return hash_.isNonZero(); + } + int + signum() const + { + return hash_.signum(); + } + void + zero() + { + hash_.zero(); + } + + friend bool + operator==(SHAMapHash const& x, SHAMapHash const& y) + { + return x.hash_ == y.hash_; + } + + friend bool + operator<(SHAMapHash const& x, SHAMapHash const& y) + { + return x.hash_ < y.hash_; + } + + friend std::ostream& + operator<<(std::ostream& os, SHAMapHash const& x) + { + return os << x.hash_; + } + + friend std::string + to_string(SHAMapHash const& x) + { + return to_string(x.hash_); + } + + template + friend void + hash_append(H& h, SHAMapHash const& x) + { + hash_append(h, x.hash_); + } +}; + +inline bool +operator!=(SHAMapHash const& x, SHAMapHash const& y) +{ + return !(x == y); +} + +} // namespace ripple + +#endif // RIPPLE_BASICS_SHAMAP_HASH_H_INCLUDED diff --git a/src/ripple/basics/TaggedCache.h b/src/ripple/basics/TaggedCache.h index 157c7c8c6..45f069bd9 100644 --- a/src/ripple/basics/TaggedCache.h +++ b/src/ripple/basics/TaggedCache.h @@ -25,8 +25,11 @@ #include #include #include +#include #include #include +#include +#include #include namespace ripple { @@ -46,6 +49,7 @@ namespace ripple { template < class Key, class T, + bool IsKeyCache = false, class Hash = hardened_hash<>, class KeyEqual = std::equal_to, class Mutex = std::recursive_mutex> @@ -89,11 +93,12 @@ public: return m_clock; } - int - getTargetSize() const + /** Returns the number of items in the container. */ + std::size_t + size() const { std::lock_guard lock(m_mutex); - return m_target_size; + return m_cache.size(); } void @@ -103,8 +108,15 @@ public: m_target_size = s; if (s > 0) - m_cache.rehash(static_cast( - (s + (s >> 2)) / m_cache.max_load_factor() + 1)); + { + for (auto& partition : m_cache.map()) + { + partition.rehash(static_cast( + (s + (s >> 2)) / + (partition.max_load_factor() * m_cache.partitions()) + + 1)); + } + } JLOG(m_journal.debug()) << m_name << " target size set to " << s; } @@ -165,22 +177,39 @@ public: m_misses = 0; } + /** Refresh the last access time on a key if present. + @return `true` If the key was found. + */ + template + bool + touch_if_exists(KeyComparable const& key) + { + std::lock_guard lock(m_mutex); + auto const iter(m_cache.find(key)); + if (iter == m_cache.end()) + { + ++m_stats.misses; + return false; + } + iter->second.touch(m_clock.now()); + ++m_stats.hits; + return true; + } + void sweep() { - int cacheRemovals = 0; - int mapRemovals = 0; - int cc = 0; - // Keep references to all the stuff we sweep - // so that we can destroy them outside the lock. - // - std::vector> stuffToSweep; + // For performance, each worker thread should exit before the swept data + // is destroyed but still within the main cache lock. + std::vector>> allStuffToSweep( + m_cache.partitions()); + clock_type::time_point const now(m_clock.now()); + clock_type::time_point when_expire; + + auto const start = std::chrono::steady_clock::now(); { - clock_type::time_point const now(m_clock.now()); - clock_type::time_point when_expire; - std::lock_guard lock(m_mutex); if (m_target_size == 0 || @@ -204,61 +233,33 @@ public: << m_target_age.count(); } - stuffToSweep.reserve(m_cache.size()); + std::vector workers; + workers.reserve(m_cache.partitions()); + std::atomic allRemovals = 0; - auto cit = m_cache.begin(); - - while (cit != m_cache.end()) + for (std::size_t p = 0; p < m_cache.partitions(); ++p) { - if (cit->second.isWeak()) - { - // weak - if (cit->second.isExpired()) - { - ++mapRemovals; - cit = m_cache.erase(cit); - } - else - { - ++cit; - } - } - else if (cit->second.last_access <= when_expire) - { - // strong, expired - --m_cache_count; - ++cacheRemovals; - if (cit->second.ptr.unique()) - { - stuffToSweep.push_back(cit->second.ptr); - ++mapRemovals; - cit = m_cache.erase(cit); - } - else - { - // remains weakly cached - cit->second.ptr.reset(); - ++cit; - } - } - else - { - // strong, not expired - ++cc; - ++cit; - } + workers.push_back(sweepHelper( + when_expire, + now, + m_cache.map()[p], + allStuffToSweep[p], + allRemovals, + lock)); } - } + for (std::thread& worker : workers) + worker.join(); - if (mapRemovals || cacheRemovals) - { - JLOG(m_journal.trace()) - << m_name << ": cache = " << m_cache.size() << "-" - << cacheRemovals << ", map-=" << mapRemovals; + m_cache_count -= allRemovals; } - - // At this point stuffToSweep will go out of scope outside the lock + // At this point allStuffToSweep will go out of scope outside the lock // and decrement the reference count on each strong pointer. + JLOG(m_journal.debug()) + << m_name << " TaggedCache sweep lock duration " + << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() + << "ms"; } bool @@ -391,51 +392,41 @@ public: std::shared_ptr fetch(const key_type& key) { - // fetch us a shared pointer to the stored data object - std::lock_guard lock(m_mutex); - - auto cit = m_cache.find(key); - - if (cit == m_cache.end()) - { + std::lock_guard l(m_mutex); + auto ret = initialFetch(key, l); + if (!ret) ++m_misses; - return {}; - } - - Entry& entry = cit->second; - entry.touch(m_clock.now()); - - if (entry.isCached()) - { - ++m_hits; - return entry.ptr; - } - - entry.ptr = entry.lock(); - - if (entry.isCached()) - { - // independent of cache size, so not counted as a hit - ++m_cache_count; - return entry.ptr; - } - - m_cache.erase(cit); - ++m_misses; - return {}; + return ret; } /** Insert the element into the container. If the key already exists, nothing happens. @return `true` If the element was inserted */ - bool + template + auto insert(key_type const& key, T const& value) + -> std::enable_if_t { auto p = std::make_shared(std::cref(value)); return canonicalize_replace_client(key, p); } + template + auto + insert(key_type const& key) -> std::enable_if_t + { + std::lock_guard lock(m_mutex); + clock_type::time_point const now(m_clock.now()); + auto [it, inserted] = m_cache.emplace( + std::piecewise_construct, + std::forward_as_tuple(key), + std::forward_as_tuple(now)); + if (!inserted) + it->second.last_access = now; + return inserted; + } + // VFALCO NOTE It looks like this returns a copy of the data in // the output parameter 'data'. This could be expensive. // Perhaps it should work like standard containers, which @@ -454,53 +445,6 @@ public: return true; } - /** Refresh the expiration time on a key. - - @param key The key to refresh. - @return `true` if the key was found and the object is cached. - */ - bool - refreshIfPresent(const key_type& key) - { - bool found = false; - - // If present, make current in cache - std::lock_guard lock(m_mutex); - - if (auto cit = m_cache.find(key); cit != m_cache.end()) - { - Entry& entry = cit->second; - - if (!entry.isCached()) - { - // Convert weak to strong. - entry.ptr = entry.lock(); - - if (entry.isCached()) - { - // We just put the object back in cache - ++m_cache_count; - entry.touch(m_clock.now()); - found = true; - } - else - { - // Couldn't get strong pointer, - // object fell out of the cache so remove the entry. - m_cache.erase(cit); - } - } - else - { - // It's cached so update the timer - entry.touch(m_clock.now()); - found = true; - } - } - - return found; - } - mutex_type& peekMutex() { @@ -522,7 +466,75 @@ public: return v; } + // CachedSLEs functions. + /** Returns the fraction of cache hits. */ + double + rate() const + { + std::lock_guard lock(m_mutex); + auto const tot = m_hits + m_misses; + if (tot == 0) + return 0; + return double(m_hits) / tot; + } + + /** Fetch an item from the cache. + If the digest was not found, Handler + will be called with this signature: + std::shared_ptr(void) + */ + template + std::shared_ptr + fetch(key_type const& digest, Handler const& h) + { + { + std::lock_guard l(m_mutex); + if (auto ret = initialFetch(digest, l)) + return ret; + } + + auto sle = h(); + if (!sle) + return {}; + + std::lock_guard l(m_mutex); + ++m_misses; + auto const [it, inserted] = + m_cache.emplace(digest, Entry(m_clock.now(), std::move(sle))); + if (!inserted) + it->second.touch(m_clock.now()); + return it->second.ptr; + } + // End CachedSLEs functions. + private: + std::shared_ptr + initialFetch(key_type const& key, std::lock_guard const& l) + { + auto cit = m_cache.find(key); + if (cit == m_cache.end()) + return {}; + + Entry& entry = cit->second; + if (entry.isCached()) + { + ++m_hits; + entry.touch(m_clock.now()); + return entry.ptr; + } + entry.ptr = entry.lock(); + if (entry.isCached()) + { + // independent of cache size, so not counted as a hit + ++m_cache_count; + entry.touch(m_clock.now()); + return entry.ptr; + } + + m_cache.erase(cit); + return {}; + } + void collect_metrics() { @@ -551,22 +563,44 @@ private: : hook(collector->make_hook(handler)) , size(collector->make_gauge(prefix, "size")) , hit_rate(collector->make_gauge(prefix, "hit_rate")) + , hits(0) + , misses(0) { } beast::insight::Hook hook; beast::insight::Gauge size; beast::insight::Gauge hit_rate; + + std::size_t hits; + std::size_t misses; }; - class Entry + class KeyOnlyEntry + { + public: + clock_type::time_point last_access; + + explicit KeyOnlyEntry(clock_type::time_point const& last_access_) + : last_access(last_access_) + { + } + + void + touch(clock_type::time_point const& now) + { + last_access = now; + } + }; + + class ValueEntry { public: std::shared_ptr ptr; std::weak_ptr weak_ptr; clock_type::time_point last_access; - Entry( + ValueEntry( clock_type::time_point const& last_access_, std::shared_ptr const& ptr_) : ptr(ptr_), weak_ptr(ptr_), last_access(last_access_) @@ -600,7 +634,136 @@ private: } }; - using cache_type = hardened_hash_map; + typedef + typename std::conditional::type + Entry; + + using KeyOnlyCacheType = + hardened_partitioned_hash_map; + + using KeyValueCacheType = + hardened_partitioned_hash_map; + + using cache_type = + hardened_partitioned_hash_map; + + [[nodiscard]] std::thread + sweepHelper( + clock_type::time_point const& when_expire, + [[maybe_unused]] clock_type::time_point const& now, + typename KeyValueCacheType::map_type& partition, + std::vector>& stuffToSweep, + std::atomic& allRemovals, + std::lock_guard const& lock) + { + return std::thread([&, this]() { + int cacheRemovals = 0; + int mapRemovals = 0; + + // Keep references to all the stuff we sweep + // so that we can destroy them outside the lock. + stuffToSweep.reserve(partition.size()); + { + auto cit = partition.begin(); + while (cit != partition.end()) + { + if (cit->second.isWeak()) + { + // weak + if (cit->second.isExpired()) + { + ++mapRemovals; + cit = partition.erase(cit); + } + else + { + ++cit; + } + } + else if (cit->second.last_access <= when_expire) + { + // strong, expired + ++cacheRemovals; + if (cit->second.ptr.unique()) + { + stuffToSweep.push_back(cit->second.ptr); + ++mapRemovals; + cit = partition.erase(cit); + } + else + { + // remains weakly cached + cit->second.ptr.reset(); + ++cit; + } + } + else + { + // strong, not expired + ++cit; + } + } + } + + if (mapRemovals || cacheRemovals) + { + JLOG(m_journal.debug()) + << "TaggedCache partition sweep " << m_name + << ": cache = " << partition.size() << "-" << cacheRemovals + << ", map-=" << mapRemovals; + } + + allRemovals += cacheRemovals; + }); + } + + [[nodiscard]] std::thread + sweepHelper( + clock_type::time_point const& when_expire, + clock_type::time_point const& now, + typename KeyOnlyCacheType::map_type& partition, + std::vector>& stuffToSweep, + std::atomic& allRemovals, + std::lock_guard const& lock) + { + return std::thread([&, this]() { + int cacheRemovals = 0; + int mapRemovals = 0; + + // Keep references to all the stuff we sweep + // so that we can destroy them outside the lock. + stuffToSweep.reserve(partition.size()); + { + auto cit = partition.begin(); + while (cit != partition.end()) + { + if (cit->second.last_access > now) + { + cit->second.last_access = now; + ++cit; + } + else if (cit->second.last_access <= when_expire) + { + cit = partition.erase(cit); + } + else + { + ++cit; + } + } + } + + if (mapRemovals || cacheRemovals) + { + JLOG(m_journal.debug()) + << "TaggedCache partition sweep " << m_name + << ": cache = " << partition.size() << "-" << cacheRemovals + << ", map-=" << mapRemovals; + } + + allRemovals += cacheRemovals; + }); + }; beast::Journal m_journal; clock_type& m_clock; diff --git a/src/ripple/basics/UnorderedContainers.h b/src/ripple/basics/UnorderedContainers.h index 2758551b9..e929ebec8 100644 --- a/src/ripple/basics/UnorderedContainers.h +++ b/src/ripple/basics/UnorderedContainers.h @@ -21,6 +21,7 @@ #define RIPPLE_BASICS_UNORDEREDCONTAINERS_H_INCLUDED #include +#include #include #include #include @@ -86,6 +87,15 @@ template < class Allocator = std::allocator>> using hardened_hash_map = std::unordered_map; +template < + class Key, + class Value, + class Hash = hardened_hash, + class Pred = std::equal_to, + class Allocator = std::allocator>> +using hardened_partitioned_hash_map = + partitioned_unordered_map; + template < class Key, class Value, diff --git a/src/ripple/basics/impl/partitioned_unordered_map.cpp b/src/ripple/basics/impl/partitioned_unordered_map.cpp new file mode 100644 index 000000000..6fb2cbec1 --- /dev/null +++ b/src/ripple/basics/impl/partitioned_unordered_map.cpp @@ -0,0 +1,78 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include + +#include +#include +#include +#include +#include +#include + +namespace ripple { + +static std::size_t +extract(uint256 const& key) +{ + return *reinterpret_cast(key.data()); +} + +static std::size_t +extract(SHAMapHash const& key) +{ + return *reinterpret_cast(key.as_uint256().data()); +} + +static std::size_t +extract(LedgerIndex key) +{ + return static_cast(key); +} + +static std::size_t +extract(std::string const& key) +{ + return ::beast::uhash<>{}(key); +} + +template +std::size_t +partitioner(Key const& key, std::size_t const numPartitions) +{ + return extract(key) % numPartitions; +} + +template std::size_t +partitioner( + LedgerIndex const& key, + std::size_t const numPartitions); + +template std::size_t +partitioner(uint256 const& key, std::size_t const numPartitions); + +template std::size_t +partitioner(SHAMapHash const& key, std::size_t const numPartitions); + +template std::size_t +partitioner( + std::string const& key, + std::size_t const numPartitions); + +} // namespace ripple diff --git a/src/ripple/basics/partitioned_unordered_map.h b/src/ripple/basics/partitioned_unordered_map.h new file mode 100644 index 000000000..08f4cba9d --- /dev/null +++ b/src/ripple/basics/partitioned_unordered_map.h @@ -0,0 +1,409 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2021 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_BASICS_PARTITIONED_UNORDERED_MAP_H +#define RIPPLE_BASICS_PARTITIONED_UNORDERED_MAP_H + +#include +#include +#include +#include +#include +#include +#include + +namespace ripple { + +template +std::size_t +partitioner(Key const& key, std::size_t const numPartitions); + +template < + typename Key, + typename Value, + typename Hash, + typename Pred = std::equal_to, + typename Alloc = std::allocator>> +class partitioned_unordered_map +{ + std::size_t partitions_; + +public: + using key_type = Key; + using mapped_type = Value; + using value_type = std::pair; + using size_type = std::size_t; + using difference_type = std::size_t; + using hasher = Hash; + using key_equal = Pred; + using allocator_type = Alloc; + using reference = value_type&; + using const_reference = value_type const&; + using pointer = value_type*; + using const_pointer = value_type const*; + using map_type = std:: + unordered_map; + using partition_map_type = std::vector; + + struct iterator + { + using iterator_category = std::forward_iterator_tag; + partition_map_type* map_{nullptr}; + typename partition_map_type::iterator ait_; + typename map_type::iterator mit_; + + iterator() = default; + + iterator(partition_map_type* map) : map_(map) + { + } + + reference + operator*() const + { + return *mit_; + } + + pointer + operator->() const + { + return &(*mit_); + } + + void + inc() + { + ++mit_; + while (mit_ == ait_->end()) + { + ++ait_; + if (ait_ == map_->end()) + return; + mit_ = ait_->begin(); + } + } + + // ++it + iterator& + operator++() + { + inc(); + return *this; + } + + // it++ + iterator + operator++(int) + { + iterator tmp(*this); + inc(); + return tmp; + } + + friend bool + operator==(iterator const& lhs, iterator const& rhs) + { + return lhs.map_ == rhs.map_ && lhs.ait_ == rhs.ait_ && + lhs.mit_ == rhs.mit_; + } + + friend bool + operator!=(iterator const& lhs, iterator const& rhs) + { + return !(lhs == rhs); + } + }; + + struct const_iterator + { + using iterator_category = std::forward_iterator_tag; + + partition_map_type* map_{nullptr}; + typename partition_map_type::iterator ait_; + typename map_type::iterator mit_; + + const_iterator() = default; + + const_iterator(partition_map_type* map) : map_(map) + { + } + + const_iterator(iterator const& orig) + { + map_ = orig.map_; + ait_ = orig.ait_; + mit_ = orig.mit_; + } + + const_reference + operator*() const + { + return *mit_; + } + + const_pointer + operator->() const + { + return &(*mit_); + } + + void + inc() + { + ++mit_; + while (mit_ == ait_->end()) + { + ++ait_; + if (ait_ == map_->end()) + return; + mit_ = ait_->begin(); + } + } + + // ++it + const_iterator& + operator++() + { + inc(); + return *this; + } + + // it++ + const_iterator + operator++(int) + { + const_iterator tmp(*this); + inc(); + return tmp; + } + + friend bool + operator==(const_iterator const& lhs, const_iterator const& rhs) + { + return lhs.map_ == rhs.map_ && lhs.ait_ == rhs.ait_ && + lhs.mit_ == rhs.mit_; + } + + friend bool + operator!=(const_iterator const& lhs, const_iterator const& rhs) + { + return !(lhs == rhs); + } + }; + +private: + std::size_t + partitioner(Key const& key) const + { + return ripple::partitioner(key, partitions_); + } + + template + static void + end(T& it) + { + it.ait_ = it.map_->end(); + it.mit_ = it.map_->back().end(); + } + + template + static void + begin(T& it) + { + for (it.ait_ = it.map_->begin(); it.ait_ != it.map_->end(); ++it.ait_) + { + if (it.ait_->begin() == it.ait_->end()) + continue; + it.mit_ = it.ait_->begin(); + return; + } + end(it); + } + +public: + partitioned_unordered_map( + std::optional partitions = std::nullopt) + { + // Set partitions to the number of hardware threads if the parameter + // is either empty or set to 0. + partitions_ = partitions && *partitions + ? *partitions + : std::thread::hardware_concurrency(); + map_.resize(partitions_); + assert(partitions_); + } + + std::size_t + partitions() const + { + return partitions_; + } + + partition_map_type& + map() + { + return map_; + } + + iterator + begin() + { + iterator it(&map_); + begin(it); + return it; + } + + const_iterator + cbegin() const + { + const_iterator it(&map_); + begin(it); + return it; + } + + const_iterator + begin() const + { + return cbegin(); + } + + iterator + end() + { + iterator it(&map_); + end(it); + return it; + } + + const_iterator + cend() const + { + const_iterator it(&map_); + end(it); + return it; + } + + const_iterator + end() const + { + return cend(); + } + +private: + template + void + find(key_type const& key, T& it) const + { + it.ait_ = it.map_->begin() + partitioner(key); + it.mit_ = it.ait_->find(key); + if (it.mit_ == it.ait_->end()) + end(it); + } + +public: + iterator + find(key_type const& key) + { + iterator it(&map_); + find(key, it); + return it; + } + + const_iterator + find(key_type const& key) const + { + const_iterator it(&map_); + find(key, it); + return it; + } + + template + std::pair + emplace(std::piecewise_construct_t const&, T&& keyTuple, U&& valueTuple) + { + auto const& key = std::get<0>(keyTuple); + iterator it(&map_); + it.ait_ = it.map_->begin() + partitioner(key); + auto [eit, inserted] = it.ait_->emplace( + std::piecewise_construct, + std::forward(keyTuple), + std::forward(valueTuple)); + it.mit_ = eit; + return {it, inserted}; + } + + template + std::pair + emplace(T&& key, U&& val) + { + iterator it(&map_); + it.ait_ = it.map_->begin() + partitioner(key); + auto [eit, inserted] = + it.ait_->emplace(std::forward(key), std::forward(val)); + it.mit_ = eit; + return {it, inserted}; + } + + void + clear() + { + for (auto& p : map_) + p.clear(); + } + + iterator + erase(const_iterator position) + { + iterator it(&map_); + it.ait_ = position.ait_; + it.mit_ = position.ait_->erase(position.mit_); + + while (it.mit_ == it.ait_->end()) + { + ++it.ait_; + if (it.ait_ == it.map_->end()) + break; + it.mit_ = it.ait_->begin(); + } + + return it; + } + + std::size_t + size() const + { + std::size_t ret = 0; + for (auto& p : map_) + ret += p.size(); + return ret; + } + + Value& + operator[](Key const& key) + { + return map_[partitioner(key)][key]; + } + +private: + mutable partition_map_type map_{}; +}; + +} // namespace ripple + +#endif // RIPPLE_BASICS_PARTITIONED_UNORDERED_MAP_H diff --git a/src/ripple/consensus/Consensus.h b/src/ripple/consensus/Consensus.h index 7a5c49c85..a86144c38 100644 --- a/src/ripple/consensus/Consensus.h +++ b/src/ripple/consensus/Consensus.h @@ -666,6 +666,7 @@ Consensus::startRoundInternal( ConsensusMode mode) { phase_ = ConsensusPhase::open; + JLOG(j_.debug()) << "transitioned to ConsensusPhase::open"; mode_.set(mode, adaptor_); now_ = now; prevLedgerID_ = prevLedgerID; @@ -1290,6 +1291,7 @@ Consensus::phaseEstablish() prevProposers_ = currPeerPositions_.size(); prevRoundTime_ = result_->roundTime.read(); phase_ = ConsensusPhase::accepted; + JLOG(j_.debug()) << "transitioned to ConsensusPhase::accepted"; adaptor_.onAccept( *result_, previousLedger_, @@ -1307,6 +1309,7 @@ Consensus::closeLedger() assert(!result_); phase_ = ConsensusPhase::establish; + JLOG(j_.debug()) << "transitioned to ConsensusPhase::establish"; rawCloseTimes_.self = now_; result_.emplace(adaptor_.onClose(previousLedger_, now_, mode_.get())); diff --git a/src/ripple/consensus/Validations.h b/src/ripple/consensus/Validations.h index 2cbddaa51..9200ac883 100644 --- a/src/ripple/consensus/Validations.h +++ b/src/ripple/consensus/Validations.h @@ -722,46 +722,59 @@ public: validationSET_EXPIRES ago and were not asked to keep. */ void - expire() + expire(beast::Journal& j) { - std::lock_guard lock{mutex_}; - if (toKeep_) + auto const start = std::chrono::steady_clock::now(); { - // We only need to refresh the keep range when it's just about to - // expire. Track the next time we need to refresh. - static std::chrono::steady_clock::time_point refreshTime; - if (auto const now = byLedger_.clock().now(); refreshTime <= now) + std::lock_guard lock{mutex_}; + if (toKeep_) { - // The next refresh time is shortly before the expiration - // time from now. - refreshTime = now + parms_.validationSET_EXPIRES - - parms_.validationFRESHNESS; - - for (auto i = byLedger_.begin(); i != byLedger_.end(); ++i) + // We only need to refresh the keep range when it's just about + // to expire. Track the next time we need to refresh. + static std::chrono::steady_clock::time_point refreshTime; + if (auto const now = byLedger_.clock().now(); + refreshTime <= now) { - auto const& validationMap = i->second; - if (!validationMap.empty()) + // The next refresh time is shortly before the expiration + // time from now. + refreshTime = now + parms_.validationSET_EXPIRES - + parms_.validationFRESHNESS; + + for (auto i = byLedger_.begin(); i != byLedger_.end(); ++i) { - auto const seq = validationMap.begin()->second.seq(); - if (toKeep_->low_ <= seq && seq < toKeep_->high_) + auto const& validationMap = i->second; + if (!validationMap.empty()) { - byLedger_.touch(i); + auto const seq = + validationMap.begin()->second.seq(); + if (toKeep_->low_ <= seq && seq < toKeep_->high_) + { + byLedger_.touch(i); + } + } + } + + for (auto i = bySequence_.begin(); i != bySequence_.end(); + ++i) + { + if (toKeep_->low_ <= i->first && + i->first < toKeep_->high_) + { + bySequence_.touch(i); } } } - - for (auto i = bySequence_.begin(); i != bySequence_.end(); ++i) - { - if (toKeep_->low_ <= i->first && i->first < toKeep_->high_) - { - bySequence_.touch(i); - } - } } - } - beast::expire(byLedger_, parms_.validationSET_EXPIRES); - beast::expire(bySequence_, parms_.validationSET_EXPIRES); + beast::expire(byLedger_, parms_.validationSET_EXPIRES); + beast::expire(bySequence_, parms_.validationSET_EXPIRES); + } + JLOG(j.debug()) + << "Validations sets sweep lock duration " + << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count() + << "ms"; } /** Update trust status of validations diff --git a/src/ripple/ledger/CachedSLEs.h b/src/ripple/ledger/CachedSLEs.h index 190bc3709..d2b04e2cb 100644 --- a/src/ripple/ledger/CachedSLEs.h +++ b/src/ripple/ledger/CachedSLEs.h @@ -20,90 +20,12 @@ #ifndef RIPPLE_LEDGER_CACHEDSLES_H_INCLUDED #define RIPPLE_LEDGER_CACHEDSLES_H_INCLUDED -#include -#include +#include +#include #include -#include -#include namespace ripple { +using CachedSLEs = TaggedCache; +} -/** Caches SLEs by their digest. */ -class CachedSLEs -{ -public: - using digest_type = uint256; - - using value_type = std::shared_ptr; - - CachedSLEs(CachedSLEs const&) = delete; - CachedSLEs& - operator=(CachedSLEs const&) = delete; - - template - CachedSLEs( - std::chrono::duration const& timeToLive, - Stopwatch& clock) - : timeToLive_(timeToLive), map_(clock) - { - } - - /** Discard expired entries. - - Needs to be called periodically. - */ - void - expire(); - - /** Fetch an item from the cache. - - If the digest was not found, Handler - will be called with this signature: - - std::shared_ptr(void) - */ - template - value_type - fetch(digest_type const& digest, Handler const& h) - { - { - std::lock_guard lock(mutex_); - auto iter = map_.find(digest); - if (iter != map_.end()) - { - ++hit_; - map_.touch(iter); - return iter->second; - } - } - auto sle = h(); - if (!sle) - return nullptr; - std::lock_guard lock(mutex_); - ++miss_; - auto const [it, inserted] = map_.emplace(digest, std::move(sle)); - if (!inserted) - map_.touch(it); - return it->second; - } - - /** Returns the fraction of cache hits. */ - double - rate() const; - -private: - std::size_t hit_ = 0; - std::size_t miss_ = 0; - std::mutex mutable mutex_; - Stopwatch::duration timeToLive_; - beast::aged_unordered_map< - digest_type, - value_type, - Stopwatch::clock_type, - hardened_hash> - map_; -}; - -} // namespace ripple - -#endif +#endif // RIPPLE_LEDGER_CACHEDSLES_H_INCLUDED diff --git a/src/ripple/ledger/impl/CachedSLEs.cpp b/src/ripple/ledger/impl/CachedSLEs.cpp deleted file mode 100644 index 3d00aeaf1..000000000 --- a/src/ripple/ledger/impl/CachedSLEs.cpp +++ /dev/null @@ -1,57 +0,0 @@ -//------------------------------------------------------------------------------ -/* - This file is part of rippled: https://github.com/ripple/rippled - Copyright (c) 2012, 2013 Ripple Labs Inc. - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted, provided that the above - copyright notice and this permission notice appear in all copies. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -*/ -//============================================================================== - -#include -#include - -namespace ripple { - -void -CachedSLEs::expire() -{ - std::vector> trash; - { - auto const expireTime = map_.clock().now() - timeToLive_; - std::lock_guard lock(mutex_); - for (auto iter = map_.chronological.begin(); - iter != map_.chronological.end(); - ++iter) - { - if (iter.when() > expireTime) - break; - if (iter->second.unique()) - { - trash.emplace_back(std::move(iter->second)); - iter = map_.erase(iter); - } - } - } -} - -double -CachedSLEs::rate() const -{ - std::lock_guard lock(mutex_); - auto const tot = hit_ + miss_; - if (tot == 0) - return 0; - return double(hit_) / tot; -} - -} // namespace ripple diff --git a/src/ripple/nodestore/Database.h b/src/ripple/nodestore/Database.h index 471621bc8..c2c5b5b88 100644 --- a/src/ripple/nodestore/Database.h +++ b/src/ripple/nodestore/Database.h @@ -20,7 +20,6 @@ #ifndef RIPPLE_NODESTORE_DATABASE_H_INCLUDED #define RIPPLE_NODESTORE_DATABASE_H_INCLUDED -#include #include #include #include diff --git a/src/ripple/nodestore/impl/Shard.h b/src/ripple/nodestore/impl/Shard.h index 4fc17a8d4..17001a6b8 100644 --- a/src/ripple/nodestore/impl/Shard.h +++ b/src/ripple/nodestore/impl/Shard.h @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -39,7 +40,7 @@ namespace ripple { namespace NodeStore { using PCache = TaggedCache; -using NCache = KeyCache; +using NCache = KeyCache; class DatabaseShard; /* A range of historical ledgers backed by a node store. diff --git a/src/ripple/shamap/FullBelowCache.h b/src/ripple/shamap/FullBelowCache.h index ad051b1eb..6d809d3b9 100644 --- a/src/ripple/shamap/FullBelowCache.h +++ b/src/ripple/shamap/FullBelowCache.h @@ -21,8 +21,10 @@ #define RIPPLE_SHAMAP_FULLBELOWCACHE_H_INCLUDED #include +#include #include #include +#include #include #include @@ -33,17 +35,15 @@ namespace detail { /** Remembers which tree keys have all descendants resident. This optimizes the process of acquiring a complete tree. */ -template class BasicFullBelowCache { private: - using CacheType = KeyCache; + using CacheType = KeyCache; public: enum { defaultCacheTargetSize = 0 }; - using key_type = Key; - using size_type = typename CacheType::size_type; + using key_type = uint256; using clock_type = typename CacheType::clock_type; /** Construct the cache. @@ -56,11 +56,12 @@ public: BasicFullBelowCache( std::string const& name, clock_type& clock, + beast::Journal j, beast::insight::Collector::ptr const& collector = beast::insight::NullCollector::New(), std::size_t target_size = defaultCacheTargetSize, std::chrono::seconds expiration = std::chrono::minutes{2}) - : m_cache(name, clock, collector, target_size, expiration), m_gen(1) + : m_cache(name, target_size, expiration, clock, j, collector), m_gen(1) { } @@ -75,7 +76,7 @@ public: Thread safety: Safe to call from any thread. */ - size_type + std::size_t size() const { return m_cache.size(); @@ -138,13 +139,13 @@ public: } private: - KeyCache m_cache; + CacheType m_cache; std::atomic m_gen; }; } // namespace detail -using FullBelowCache = detail::BasicFullBelowCache; +using FullBelowCache = detail::BasicFullBelowCache; } // namespace ripple diff --git a/src/ripple/shamap/SHAMapTreeNode.h b/src/ripple/shamap/SHAMapTreeNode.h index bc3a0b0d8..8e351cce9 100644 --- a/src/ripple/shamap/SHAMapTreeNode.h +++ b/src/ripple/shamap/SHAMapTreeNode.h @@ -21,6 +21,7 @@ #define RIPPLE_SHAMAP_SHAMAPTREENODE_H_INCLUDED #include +#include #include #include #include @@ -42,88 +43,6 @@ static constexpr unsigned char const wireTypeInner = 2; static constexpr unsigned char const wireTypeCompressedInner = 3; static constexpr unsigned char const wireTypeTransactionWithMeta = 4; -// A SHAMapHash is the hash of a node in a SHAMap, and also the -// type of the hash of the entire SHAMap. - -class SHAMapHash -{ - uint256 hash_; - -public: - SHAMapHash() = default; - explicit SHAMapHash(uint256 const& hash) : hash_(hash) - { - } - - uint256 const& - as_uint256() const - { - return hash_; - } - uint256& - as_uint256() - { - return hash_; - } - bool - isZero() const - { - return hash_.isZero(); - } - bool - isNonZero() const - { - return hash_.isNonZero(); - } - int - signum() const - { - return hash_.signum(); - } - void - zero() - { - hash_.zero(); - } - - friend bool - operator==(SHAMapHash const& x, SHAMapHash const& y) - { - return x.hash_ == y.hash_; - } - - friend bool - operator<(SHAMapHash const& x, SHAMapHash const& y) - { - return x.hash_ < y.hash_; - } - - friend std::ostream& - operator<<(std::ostream& os, SHAMapHash const& x) - { - return os << x.hash_; - } - - friend std::string - to_string(SHAMapHash const& x) - { - return to_string(x.hash_); - } - - template - friend void - hash_append(H& h, SHAMapHash const& x) - { - hash_append(h, x.hash_); - } -}; - -inline bool -operator!=(SHAMapHash const& x, SHAMapHash const& y) -{ - return !(x == y); -} - enum class SHAMapNodeType { tnINNER = 1, tnTRANSACTION_NM = 2, // transaction, no metadata diff --git a/src/ripple/shamap/impl/NodeFamily.cpp b/src/ripple/shamap/impl/NodeFamily.cpp index f81702037..f9c6dedb2 100644 --- a/src/ripple/shamap/impl/NodeFamily.cpp +++ b/src/ripple/shamap/impl/NodeFamily.cpp @@ -31,6 +31,7 @@ NodeFamily::NodeFamily(Application& app, CollectorManager& cm) , fbCache_(std::make_shared( "Node family full below cache", stopwatch(), + app.journal("NodeFamilyFulLBelowCache"), cm.collector(), fullBelowTargetSize, fullBelowExpiration)) diff --git a/src/ripple/shamap/impl/ShardFamily.cpp b/src/ripple/shamap/impl/ShardFamily.cpp index ee4a7c83c..eadfc42aa 100644 --- a/src/ripple/shamap/impl/ShardFamily.cpp +++ b/src/ripple/shamap/impl/ShardFamily.cpp @@ -55,6 +55,7 @@ ShardFamily::getFullBelowCache(std::uint32_t ledgerSeq) auto fbCache{std::make_shared( "Shard family full below cache shard " + std::to_string(shardIndex), stopwatch(), + j_, cm_.collector(), fullBelowTargetSize, fullBelowExpiration)}; diff --git a/src/test/basics/KeyCache_test.cpp b/src/test/basics/KeyCache_test.cpp index c3ee03595..7f3f13e27 100644 --- a/src/test/basics/KeyCache_test.cpp +++ b/src/test/basics/KeyCache_test.cpp @@ -17,10 +17,13 @@ */ //============================================================================== -#include +#include #include #include #include +#include +#include +#include namespace ripple { @@ -35,32 +38,31 @@ public: clock.set(0); using Key = std::string; - using Cache = KeyCache; + using Cache = TaggedCache; + + test::SuiteJournal j("KeyCacheTest", *this); // Insert an item, retrieve it, and age it so it gets purged. { - Cache c("test", clock, 1, 2s); + Cache c("test", LedgerIndex(1), 2s, clock, j); BEAST_EXPECT(c.size() == 0); BEAST_EXPECT(c.insert("one")); BEAST_EXPECT(!c.insert("one")); BEAST_EXPECT(c.size() == 1); - BEAST_EXPECT(c.exists("one")); BEAST_EXPECT(c.touch_if_exists("one")); ++clock; c.sweep(); BEAST_EXPECT(c.size() == 1); - BEAST_EXPECT(c.exists("one")); ++clock; c.sweep(); BEAST_EXPECT(c.size() == 0); - BEAST_EXPECT(!c.exists("one")); BEAST_EXPECT(!c.touch_if_exists("one")); } // Insert two items, have one expire { - Cache c("test", clock, 2, 2s); + Cache c("test", LedgerIndex(2), 2s, clock, j); BEAST_EXPECT(c.insert("one")); BEAST_EXPECT(c.size() == 1); @@ -73,12 +75,11 @@ public: ++clock; c.sweep(); BEAST_EXPECT(c.size() == 1); - BEAST_EXPECT(c.exists("two")); } // Insert three items (1 over limit), sweep { - Cache c("test", clock, 2, 3s); + Cache c("test", LedgerIndex(2), 3s, clock, j); BEAST_EXPECT(c.insert("one")); ++clock; diff --git a/src/test/basics/TaggedCache_test.cpp b/src/test/basics/TaggedCache_test.cpp index 9eb2d3cb5..6a5b44299 100644 --- a/src/test/basics/TaggedCache_test.cpp +++ b/src/test/basics/TaggedCache_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include namespace ripple { @@ -48,7 +49,7 @@ public: TestStopwatch clock; clock.set(0); - using Key = int; + using Key = LedgerIndex; using Value = std::string; using Cache = TaggedCache; diff --git a/src/test/consensus/Validations_test.cpp b/src/test/consensus/Validations_test.cpp index beb43421c..79de1fc80 100644 --- a/src/test/consensus/Validations_test.cpp +++ b/src/test/consensus/Validations_test.cpp @@ -21,9 +21,9 @@ #include #include #include -#include - #include +#include +#include #include #include #include @@ -703,6 +703,7 @@ class Validations_test : public beast::unit_test::suite { // Verify expiring clears out validations stored by ledger testcase("Expire validations"); + SuiteJournal j("Validations_test", *this); LedgerHistoryHelper h; TestHarness harness(h.oracle); Node const a = harness.makeNode(); @@ -713,10 +714,10 @@ class Validations_test : public beast::unit_test::suite Ledger const ledgerA = h["a"]; BEAST_EXPECT(ValStatus::current == harness.add(a.validate(ledgerA))); BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); - harness.vals().expire(); + harness.vals().expire(j); BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 1); harness.clock().advance(harness.parms().validationSET_EXPIRES); - harness.vals().expire(); + harness.vals().expire(j); BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerA.id()) == 0); // use setSeqToKeep to keep the validation from expire @@ -725,7 +726,7 @@ class Validations_test : public beast::unit_test::suite BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); harness.vals().setSeqToKeep(ledgerB.seq(), ledgerB.seq() + one); harness.clock().advance(harness.parms().validationSET_EXPIRES); - harness.vals().expire(); + harness.vals().expire(j); BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 1); // change toKeep harness.vals().setSeqToKeep(ledgerB.seq() + one, ledgerB.seq() + two); @@ -736,7 +737,7 @@ class Validations_test : public beast::unit_test::suite for (int i = 0; i < loops; ++i) { harness.clock().advance(harness.parms().validationFRESHNESS); - harness.vals().expire(); + harness.vals().expire(j); } BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerB.id()) == 0); @@ -746,7 +747,7 @@ class Validations_test : public beast::unit_test::suite BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 1); harness.vals().setSeqToKeep(ledgerC.seq() - one, ledgerC.seq()); harness.clock().advance(harness.parms().validationSET_EXPIRES); - harness.vals().expire(); + harness.vals().expire(j); BEAST_EXPECT(harness.vals().numTrustedForLedger(ledgerC.id()) == 0); } diff --git a/src/test/csf/Peer.h b/src/test/csf/Peer.h index 2f3ce1931..3a61b853c 100644 --- a/src/test/csf/Peer.h +++ b/src/test/csf/Peer.h @@ -919,7 +919,7 @@ struct Peer start() { // TODO: Expire validations less frequently? - validations.expire(); + validations.expire(j); scheduler.in(parms().ledgerGRANULARITY, [&]() { timerEntry(); }); startRound(); } diff --git a/src/test/shamap/common.h b/src/test/shamap/common.h index 1f6b924e0..c4238b2a6 100644 --- a/src/test/shamap/common.h +++ b/src/test/shamap/common.h @@ -46,7 +46,8 @@ public: TestNodeFamily(beast::Journal j) : fbCache_(std::make_shared( "App family full below cache", - clock_)) + clock_, + j)) , tnCache_(std::make_shared( "App family tree node cache", 65536,