diff --git a/Builds/QtCreator/rippled.pro b/Builds/QtCreator/rippled.pro index 40cf9912f..527446f41 100644 --- a/Builds/QtCreator/rippled.pro +++ b/Builds/QtCreator/rippled.pro @@ -69,6 +69,7 @@ SOURCES += \ ../../src/ripple/http/ripple_http.cpp \ ../../src/ripple/json/ripple_json.cpp \ ../../src/ripple/peerfinder/ripple_peerfinder.cpp \ + ../../src/ripple/radmap/ripple_radmap.cpp \ ../../src/ripple/resource/ripple_resource.cpp \ ../../src/ripple/rpc/ripple_rpc.cpp \ ../../src/ripple/sitefiles/ripple_sitefiles.cpp \ diff --git a/Builds/VisualStudio2012/RippleD.vcxproj b/Builds/VisualStudio2012/RippleD.vcxproj index f9eb20ed7..d9a13e45d 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj +++ b/Builds/VisualStudio2012/RippleD.vcxproj @@ -22,7 +22,19 @@ - + + true + true + true + true + + + true + true + true + true + + true true true @@ -134,6 +146,13 @@ true + + true + true + true + true + + true true @@ -1418,24 +1437,12 @@ true true - - true - true - true - true - true true true true - - true - true - true - true - true true @@ -2207,7 +2214,10 @@ - + + + + @@ -2251,6 +2261,9 @@ + + + @@ -2455,6 +2468,7 @@ + @@ -2464,6 +2478,7 @@ + @@ -2528,9 +2543,7 @@ - - @@ -2730,6 +2743,8 @@ + + diff --git a/Builds/VisualStudio2012/RippleD.vcxproj.filters b/Builds/VisualStudio2012/RippleD.vcxproj.filters index 67a300105..b75f62953 100644 --- a/Builds/VisualStudio2012/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2012/RippleD.vcxproj.filters @@ -292,20 +292,23 @@ {b99052d6-a903-4dfd-9c68-ff767a7d8f63} - - {70365f6a-d1e6-45f2-a064-0e842b0fdd78} + + {c9eed7bc-87ac-4a4f-ab79-91de2319d22a} - - {9b8137bd-737d-4825-98bf-897a0635293a} + + {8840a3b4-75b5-4bc2-9992-a8c4990ed96e} + + + {46aff75d-bfd1-47a4-b09f-e3efca4809e1} + + + {9adc00c7-d533-454e-b1ea-fd21ba41eb9a} [2] Old Ripple\ripple_basics\containers - - [2] Old Ripple\ripple_basics\containers - [2] Old Ripple\ripple_basics\utility @@ -1407,15 +1410,9 @@ [1] Ripple\common - - [1] Ripple\common\functional\impl - [2] Old Ripple\ripple_core\nodestore\backend - - [2] Old Ripple\ripple_basics\containers - [2] Old Ripple\ripple_core\nodestore\impl @@ -1452,17 +1449,26 @@ [2] Old Ripple\ripple_app\ledger + + [1] Ripple\common\impl + + + [1] Ripple\common\impl + + + [1] Ripple\common\impl + + + [1] Ripple\radmap + + + [1] Ripple\radmap\impl + - - [2] Old Ripple\ripple_basics\containers - [2] Old Ripple\ripple_basics\containers - - [2] Old Ripple\ripple_basics\containers - [2] Old Ripple\ripple_basics\system @@ -2913,9 +2919,6 @@ [1] Ripple\types\api - - [1] Ripple\common\functional - [2] Old Ripple\ripple_core\nodestore\backend @@ -2952,6 +2955,33 @@ [2] Old Ripple\ripple_app\paths + + [1] Ripple\common + + + [1] Ripple\common + + + [1] Ripple\common + + + [1] Ripple\radmap\api + + + [1] Ripple\radmap + + + [1] Ripple\common + + + [1] Ripple\radmap\api + + + [2] Old Ripple\ripple_app\main + + + [2] Old Ripple\ripple_app\main + @@ -3067,6 +3097,12 @@ [2] Old Ripple\ripple_app\ledger + + [1] Ripple\radmap + + + [1] Ripple\radmap + diff --git a/Builds/VisualStudio2013/RippleD.vcxproj b/Builds/VisualStudio2013/RippleD.vcxproj index d8234da41..dc150e7f5 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj +++ b/Builds/VisualStudio2013/RippleD.vcxproj @@ -22,7 +22,19 @@ - + + true + true + true + true + + + true + true + true + true + + true true true @@ -134,6 +146,13 @@ true + + true + true + true + true + + true true @@ -1418,24 +1437,12 @@ true true - - true - true - true - true - true true true true - - true - true - true - true - true true @@ -2207,7 +2214,10 @@ - + + + + @@ -2251,6 +2261,9 @@ + + + @@ -2455,6 +2468,7 @@ + @@ -2464,6 +2478,7 @@ + @@ -2528,10 +2543,8 @@ - - @@ -2731,6 +2744,8 @@ + + diff --git a/Builds/VisualStudio2013/RippleD.vcxproj.filters b/Builds/VisualStudio2013/RippleD.vcxproj.filters index bfa26044f..3f1e0e00e 100644 --- a/Builds/VisualStudio2013/RippleD.vcxproj.filters +++ b/Builds/VisualStudio2013/RippleD.vcxproj.filters @@ -292,11 +292,17 @@ {b99052d6-a903-4dfd-9c68-ff767a7d8f63} - - {e2a53cb6-0594-4646-a36f-68d598972da4} + + {6f3085f6-dbe3-4622-a680-682787d0708c} - - {24383350-1f23-42f8-8b58-399fee670b17} + + {1b34e7e8-8260-488c-8d09-bdd3b474f9e3} + + + {19de3695-4341-49db-9da2-b220bc9e0149} + + + {67371f65-f9be-45b1-81e8-c83ef3336e5c} @@ -1443,14 +1449,20 @@ [2] Old Ripple\ripple_app\ledger - - [1] Ripple\common\functional\impl + + [1] Ripple\radmap - - [2] Old Ripple\ripple_basics\containers + + [1] Ripple\radmap\impl - - [2] Old Ripple\ripple_basics\containers + + [1] Ripple\common\impl + + + [1] Ripple\common\impl + + + [1] Ripple\common\impl @@ -2946,14 +2958,32 @@ [2] Old Ripple\ripple_basics\containers - - [1] Ripple\common\functional + + [1] Ripple\radmap - - [2] Old Ripple\ripple_basics\containers + + [1] Ripple\radmap\api - - [2] Old Ripple\ripple_basics\containers + + [1] Ripple\radmap\api + + + [1] Ripple\common + + + [1] Ripple\common + + + [1] Ripple\common + + + [1] Ripple\common + + + [2] Old Ripple\ripple_app\main + + + [2] Old Ripple\ripple_app\main @@ -3070,6 +3100,12 @@ [2] Old Ripple\ripple_app\ledger + + [1] Ripple\radmap + + + [1] Ripple\radmap + diff --git a/SConstruct b/SConstruct index d390162d5..57bb15797 100644 --- a/SConstruct +++ b/SConstruct @@ -167,6 +167,7 @@ COMPILED_FILES.extend([ 'src/ripple/http/ripple_http.cpp', 'src/ripple/json/ripple_json.cpp', 'src/ripple/peerfinder/ripple_peerfinder.cpp', + 'src/ripple/radmap/ripple_radmap.cpp', 'src/ripple/resource/ripple_resource.cpp', 'src/ripple/rocksdb/ripple_rocksdb.cpp', 'src/ripple/rpc/ripple_rpc.cpp', diff --git a/src/ripple_basics/containers/KeyCache.h b/src/ripple/common/KeyCache.h similarity index 71% rename from src/ripple_basics/containers/KeyCache.h rename to src/ripple/common/KeyCache.h index 3487cfad2..030be2029 100644 --- a/src/ripple_basics/containers/KeyCache.h +++ b/src/ripple/common/KeyCache.h @@ -20,11 +20,14 @@ #ifndef RIPPLE_KEYCACHE_H_INCLUDED #define RIPPLE_KEYCACHE_H_INCLUDED -#include "beast/beast/chrono/abstract_clock.h" - #include #include +#include + +#include "beast/beast/chrono/abstract_clock.h" +#include "beast/beast/Insight.h" + namespace ripple { /** Maintains a cache of keys with no associated data. @@ -48,6 +51,26 @@ public: typedef abstract_clock clock_type; private: + struct Stats + { + template + Stats (std::string const& prefix, Handler const& handler, + insight::Collector::ptr const& collector) + : hook (collector->make_hook (handler)) + , size (collector->make_gauge (prefix, "size")) + , hit_rate (collector->make_gauge (prefix, "hit_rate")) + , hits (0) + , misses (0) + { } + + insight::Hook hook; + insight::Gauge size; + insight::Gauge hit_rate; + + std::size_t hits; + std::size_t misses; + }; + struct Entry { explicit Entry (clock_type::time_point const& last_access_) @@ -61,9 +84,10 @@ private: typedef std::unordered_map map_type; typedef typename map_type::iterator iterator; typedef std::lock_guard lock_guard; - + Mutex mutable m_mutex; map_type m_map; + Stats mutable m_stats; clock_type& m_clock; std::string const m_name; unsigned int m_target_size; @@ -77,10 +101,27 @@ public: @param size The initial target size. @param age The initial expiration time. */ - KeyCache (std::string const& name, - clock_type& clock, size_type target_size = 0, + KeyCache (std::string const& name, clock_type& clock, + insight::Collector::ptr const& collector, size_type target_size = 0, clock_type::rep expiration_seconds = 120) - : m_clock (clock) + : m_stats (name, + std::bind (&KeyCache::collect_metrics, this), + collector) + , m_clock (clock) + , m_name (name) + , m_target_size (target_size) + , m_target_age (std::chrono::seconds (expiration_seconds)) + { + assert (m_target_size >= 0); + } + + // VFALCO TODO Use a forwarding constructor call here + KeyCache (std::string const& name, clock_type& clock, + size_type target_size = 0, clock_type::rep expiration_seconds = 120) + : m_stats (name, + std::bind (&KeyCache::collect_metrics, this), + insight::NullCollector::New ()) + , m_clock (clock) , m_name (name) , m_target_size (target_size) , m_target_age (std::chrono::seconds (expiration_seconds)) @@ -96,6 +137,12 @@ public: return m_name; } + /** Return the clock associated with the cache. */ + clock_type& clock () + { + return m_clock; + } + /** Returns the number of items in the container. */ size_type size () const { @@ -118,7 +165,13 @@ public: { lock_guard lock (m_mutex); typename map_type::const_iterator const iter (m_map.find (key)); - return iter != m_map.end (); + if (iter != m_map.end ()) + { + ++m_stats.hits; + return true; + } + ++m_stats.misses; + return false; } /** Insert the specified key. @@ -149,8 +202,12 @@ public: lock_guard lock (m_mutex); iterator const iter (m_map.find (key)); if (iter == m_map.end ()) + { + ++m_stats.misses; return false; + } iter->second.last_access = m_clock.now (); + ++m_stats.hits; return true; } @@ -161,7 +218,13 @@ public: bool erase (key_type const& key) { lock_guard lock (m_mutex); - return m_map.erase (key) > 0; + if (m_map.erase (key) > 0) + { + ++m_stats.hits; + return true; + } + ++m_stats.misses; + return false; } /** Remove stale entries from the cache. */ @@ -207,6 +270,23 @@ public: } } } + +private: + void collect_metrics () + { + m_stats.size.set (size ()); + + { + insight::Gauge::value_type hit_rate (0); + { + lock_guard lock (m_mutex); + auto const total (m_stats.hits + m_stats.misses); + if (total != 0) + hit_rate = (m_stats.hits * 100) / total; + } + m_stats.hit_rate.set (hit_rate); + } + } }; } diff --git a/src/ripple_basics/containers/TaggedCache.h b/src/ripple/common/TaggedCache.h similarity index 91% rename from src/ripple_basics/containers/TaggedCache.h rename to src/ripple/common/TaggedCache.h index 3d88c3aa0..8f093dd52 100644 --- a/src/ripple_basics/containers/TaggedCache.h +++ b/src/ripple/common/TaggedCache.h @@ -23,6 +23,10 @@ #include #include +#include + +#include "beast/beast/Insight.h" + namespace ripple { // VFALCO NOTE Deprecated @@ -49,7 +53,7 @@ template < //class Allocator = std::allocator >, class Mutex = std::recursive_mutex > -class TaggedCacheType +class TaggedCache { public: typedef Mutex mutex_type; @@ -65,10 +69,14 @@ public: public: // VFALCO TODO Change expiration_seconds to clock_type::duration - TaggedCacheType (std::string const& name, int size, - clock_type::rep expiration_seconds, clock_type& clock, Journal journal) + TaggedCache (std::string const& name, int size, + clock_type::rep expiration_seconds, clock_type& clock, Journal journal, + insight::Collector::ptr const& collector = insight::NullCollector::New ()) : m_journal (journal) , m_clock (clock) + , m_stats (name, + std::bind (&TaggedCache::collect_metrics, this), + collector) , m_name (name) , m_target_size (size) , m_target_age (std::chrono::seconds (expiration_seconds)) @@ -78,6 +86,13 @@ public: { } +public: + /** Return the clock associated with the cache. */ + clock_type& clock () + { + return m_clock; + } + int getTargetSize () const { lock_guard lock (m_mutex); @@ -453,6 +468,38 @@ public: } private: + void collect_metrics () + { + m_stats.size.set (getCacheSize ()); + + { + insight::Gauge::value_type hit_rate (0); + { + lock_guard lock (m_mutex); + auto const total (m_hits + m_misses); + if (total != 0) + hit_rate = (m_hits * 100) / total; + } + m_stats.hit_rate.set (hit_rate); + } + } + +private: + struct Stats + { + template + Stats (std::string const& prefix, Handler const& handler, + insight::Collector::ptr const& collector) + : hook (collector->make_hook (handler)) + , size (collector->make_gauge (prefix, "size")) + , hit_rate (collector->make_gauge (prefix, "hit_rate")) + { } + + insight::Hook hook; + insight::Gauge size; + insight::Gauge hit_rate; + }; + class Entry { public: @@ -481,6 +528,7 @@ private: Journal m_journal; clock_type& m_clock; + Stats m_stats; mutex_type mutable m_mutex; diff --git a/src/ripple/common/functional/counted_bind.h b/src/ripple/common/counted_bind.h similarity index 100% rename from src/ripple/common/functional/counted_bind.h rename to src/ripple/common/counted_bind.h diff --git a/src/ripple_basics/containers/KeyCache.cpp b/src/ripple/common/impl/KeyCache.cpp similarity index 99% rename from src/ripple_basics/containers/KeyCache.cpp rename to src/ripple/common/impl/KeyCache.cpp index 40f58c3dd..4de523a45 100644 --- a/src/ripple_basics/containers/KeyCache.cpp +++ b/src/ripple/common/impl/KeyCache.cpp @@ -17,7 +17,7 @@ */ //============================================================================== -#include "KeyCache.h" +#include "../KeyCache.h" namespace ripple { diff --git a/src/ripple_basics/containers/TaggedCache.cpp b/src/ripple/common/impl/TaggedCache.cpp similarity index 98% rename from src/ripple_basics/containers/TaggedCache.cpp rename to src/ripple/common/impl/TaggedCache.cpp index 95ab48109..5c303e393 100644 --- a/src/ripple_basics/containers/TaggedCache.cpp +++ b/src/ripple/common/impl/TaggedCache.cpp @@ -17,6 +17,8 @@ */ //============================================================================== +#include "../TaggedCache.h" + namespace ripple { /* @@ -44,7 +46,7 @@ public: typedef int Key; typedef std::string Value; - typedef TaggedCacheType Cache; + typedef TaggedCache Cache; Cache c ("test", 1, 1, clock, j); diff --git a/src/ripple/common/functional/impl/counted_bind.cpp b/src/ripple/common/impl/counted_bind.cpp similarity index 100% rename from src/ripple/common/functional/impl/counted_bind.cpp rename to src/ripple/common/impl/counted_bind.cpp diff --git a/src/ripple/common/ripple_common.cpp b/src/ripple/common/ripple_common.cpp index 1d1795d32..1fea9fcf2 100644 --- a/src/ripple/common/ripple_common.cpp +++ b/src/ripple/common/ripple_common.cpp @@ -21,4 +21,6 @@ #include "../../beast/modules/beast_core/beast_core.h" // for UnitTest -#include "functional/impl/counted_bind.cpp" +#include "impl/counted_bind.cpp" +#include "impl/KeyCache.cpp" +#include "impl/TaggedCache.cpp" diff --git a/src/ripple/common/seconds_clock.h b/src/ripple/common/seconds_clock.h new file mode 100644 index 000000000..9552791a2 --- /dev/null +++ b/src/ripple/common/seconds_clock.h @@ -0,0 +1,38 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_SECONDS_CLOCK_H_INCLUDED +#define RIPPLE_SECONDS_CLOCK_H_INCLUDED + +#include + +#include "beast/beast/chrono/basic_seconds_clock.h" + +namespace ripple { + +/** Returns an abstract_clock optimized for counting seconds. */ +inline beast::abstract_clock & get_seconds_clock () +{ + typedef beast::basic_seconds_clock clock_type; + return beast::get_abstract_clock (); +} + +} + +#endif diff --git a/src/ripple/radmap/README.md b/src/ripple/radmap/README.md new file mode 100644 index 000000000..a27f8f89c --- /dev/null +++ b/src/ripple/radmap/README.md @@ -0,0 +1,5 @@ +# RadMap + +A RadMap is a combination radix tree and hash map. + +(Formerly known as SHAMap) diff --git a/src/ripple/radmap/TODO.md b/src/ripple/radmap/TODO.md new file mode 100644 index 000000000..d342b2f30 --- /dev/null +++ b/src/ripple/radmap/TODO.md @@ -0,0 +1 @@ +# RadMap TODO diff --git a/src/ripple/radmap/api/BasicFullBelowCache.h b/src/ripple/radmap/api/BasicFullBelowCache.h new file mode 100644 index 000000000..7de495e97 --- /dev/null +++ b/src/ripple/radmap/api/BasicFullBelowCache.h @@ -0,0 +1,117 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RADMAP_BASICFULLBELOWCACHE_H_INCLUDED +#define RIPPLE_RADMAP_BASICFULLBELOWCACHE_H_INCLUDED + +#include "beast/beast/insight/Collector.h" + +#include "../../common/KeyCache.h" + +#include "Tuning.h" + +namespace ripple { +namespace RadMap { + +/** Remembers which tree keys have all descendants resident. + This optimizes the process of acquiring a complete tree. +*/ +template +class BasicFullBelowCache +{ +private: + typedef KeyCache CacheType; + +public: + typedef Key key_type; + typedef typename CacheType::size_type size_type; + typedef typename CacheType::clock_type clock_type; + + /** Construct the cache. + + @param name A label for diagnostics and stats reporting. + @param collector The collector to use for reporting stats. + @param targetSize The cache target size. + @param targetExpirationSeconds The expiration time for items. + */ + BasicFullBelowCache (std::string const& name, clock_type& clock, + beast::insight::Collector::ptr const& collector = + beast::insight::NullCollector::New (), + std::size_t target_size = defaultCacheTargetSize, + std::size_t expiration_seconds = defaultCacheExpirationSeconds) + : m_cache (name, clock, collector, target_size, + expiration_seconds) + { + } + + /** Return the clock associated with the cache. */ + clock_type& clock() + { + return m_cache.clock (); + } + + /** Return the number of elements in the cache. + Thread safety: + Safe to call from any thread. + */ + size_type size () const + { + return m_cache.size (); + } + + /** Remove expired cache items. + Thread safety: + Safe to call from any thread. + */ + void sweep () + { + m_cache.sweep (); + } + + /** Refresh the last access time of an item, if it exists. + Thread safety: + Safe to call from any thread. + @param key The key to refresh. + @return `true` If the key exists. + */ + bool touch_if_exists (key_type const& key) + { + return m_cache.touch_if_exists (key); + } + + /** Insert a key into the cache. + If the key already exists, the last access time will still + be refreshed. + Thread safety: + Safe to call from any thread. + @param key The key to insert. + */ + void insert (key_type const& key) + { + m_cache.insert (key); + } + +private: + KeyCache m_cache; +}; + +} +} + +#endif diff --git a/src/ripple/radmap/api/Tuning.h b/src/ripple/radmap/api/Tuning.h new file mode 100644 index 000000000..db0ad9263 --- /dev/null +++ b/src/ripple/radmap/api/Tuning.h @@ -0,0 +1,36 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RADMAP_TUNING_H_INCLUDED +#define RIPPLE_RADMAP_TUNING_H_INCLUDED + +namespace ripple { +namespace RadMap { + +enum +{ + defaultCacheTargetSize = 0 + + ,defaultCacheExpirationSeconds = 120 +}; + +} +} + +#endif diff --git a/src/ripple/radmap/impl/BasicFullBelowCache.cpp b/src/ripple/radmap/impl/BasicFullBelowCache.cpp new file mode 100644 index 000000000..9b2ddc7f7 --- /dev/null +++ b/src/ripple/radmap/impl/BasicFullBelowCache.cpp @@ -0,0 +1,28 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + + +namespace ripple { +namespace RadMap { + +// Unit test to go here if the class becomes +// a non-trivial wrapper for KeyCache. + +} +} diff --git a/src/ripple/radmap/ripple_radmap.cpp b/src/ripple/radmap/ripple_radmap.cpp new file mode 100644 index 000000000..3d39af294 --- /dev/null +++ b/src/ripple/radmap/ripple_radmap.cpp @@ -0,0 +1,22 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#include "BeastConfig.h" + +#include "impl/BasicFullBelowCache.cpp" diff --git a/src/ripple/radmap/ripple_radmap.h b/src/ripple/radmap/ripple_radmap.h new file mode 100644 index 000000000..ac3f2365d --- /dev/null +++ b/src/ripple/radmap/ripple_radmap.h @@ -0,0 +1,26 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_RADMAP_H_INCLUDED +#define RIPPLE_RADMAP_H_INCLUDED + +#include "api/BasicFullBelowCache.h" +#include "api/Tuning.h" + +#endif diff --git a/src/ripple/resource/impl/Manager.cpp b/src/ripple/resource/impl/Manager.cpp index 3b68806da..1376161cd 100644 --- a/src/ripple/resource/impl/Manager.cpp +++ b/src/ripple/resource/impl/Manager.cpp @@ -17,8 +17,6 @@ */ //============================================================================== -#include "beast/beast/make_unique.h" - namespace ripple { namespace Resource { @@ -34,9 +32,7 @@ public: Journal journal) : Thread ("Resource::Manager") , m_journal (journal) - , m_logic (collector, - get_abstract_clock (), - journal) + , m_logic (collector, get_seconds_clock (), journal) { startThread (); } diff --git a/src/ripple/resource/ripple_resource.cpp b/src/ripple/resource/ripple_resource.cpp index d9a135f91..582872eba 100644 --- a/src/ripple/resource/ripple_resource.cpp +++ b/src/ripple/resource/ripple_resource.cpp @@ -22,12 +22,14 @@ #include "ripple_resource.h" #include "../algorithm/api/DecayingSample.h" +#include "../common/seconds_clock.h" #include "beast/modules/beast_core/system/BeforeBoost.h" #include #include #include "beast/beast/Insight.h" +#include "beast/beast/make_unique.h" #include "impl/Fees.cpp" # include "impl/Kind.h" diff --git a/src/ripple_app/consensus/LedgerConsensus.cpp b/src/ripple_app/consensus/LedgerConsensus.cpp index c7b015fd0..0cebe4ed4 100644 --- a/src/ripple_app/consensus/LedgerConsensus.cpp +++ b/src/ripple_app/consensus/LedgerConsensus.cpp @@ -275,8 +275,8 @@ public: { if (hash.isZero ()) { - SHAMap::pointer empty - = boost::make_shared (smtTRANSACTION); + SHAMap::pointer empty = boost::make_shared ( + smtTRANSACTION, std::ref (getApp().getFullBelowCache())); mapComplete (hash, empty, false); return empty; } diff --git a/src/ripple_app/ledger/AcceptedLedger.cpp b/src/ripple_app/ledger/AcceptedLedger.cpp index f099baf56..99b52c113 100644 --- a/src/ripple_app/ledger/AcceptedLedger.cpp +++ b/src/ripple_app/ledger/AcceptedLedger.cpp @@ -20,10 +20,9 @@ // VFALCO TODO Remove this global and make it a member of the App // Use a dependency injection to give AcceptedLedger access. // -TaggedCacheType AcceptedLedger::s_cache ( - "AcceptedLedger", 4, 60, - get_abstract_clock (), - LogPartition::getJournal ()); +TaggedCache AcceptedLedger::s_cache ( + "AcceptedLedger", 4, 60, get_seconds_clock (), + LogPartition::getJournal ()); AcceptedLedger::AcceptedLedger (Ledger::ref ledger) : mLedger (ledger) { diff --git a/src/ripple_app/ledger/AcceptedLedger.h b/src/ripple_app/ledger/AcceptedLedger.h index 9b59cbc01..2db170378 100644 --- a/src/ripple_app/ledger/AcceptedLedger.h +++ b/src/ripple_app/ledger/AcceptedLedger.h @@ -83,7 +83,7 @@ private: void insert (AcceptedLedgerTx::ref); private: - static TaggedCacheType s_cache; + static TaggedCache s_cache; Ledger::pointer mLedger; map_t mMap; diff --git a/src/ripple_app/ledger/Ledger.cpp b/src/ripple_app/ledger/Ledger.cpp index 33afecf9b..7bb68884c 100644 --- a/src/ripple_app/ledger/Ledger.cpp +++ b/src/ripple_app/ledger/Ledger.cpp @@ -36,8 +36,10 @@ Ledger::Ledger (const RippleAddress& masterID, uint64 startAmount) , mValidHash (false) , mAccepted (false) , mImmutable (false) - , mTransactionMap (boost::make_shared (smtTRANSACTION)) - , mAccountStateMap (boost::make_shared (smtSTATE)) + , mTransactionMap (boost::make_shared (smtTRANSACTION, + std::ref (getApp().getFullBelowCache()))) + , mAccountStateMap (boost::make_shared (smtSTATE, + std::ref (getApp().getFullBelowCache()))) { // special case: put coins in root account AccountState::pointer startAccount = boost::make_shared (masterID); @@ -81,8 +83,10 @@ Ledger::Ledger (uint256 const& parentHash, , mValidHash (false) , mAccepted (false) , mImmutable (true) - , mTransactionMap (boost::make_shared (smtTRANSACTION, transHash)) - , mAccountStateMap (boost::make_shared (smtSTATE, accountHash)) + , mTransactionMap (boost::make_shared ( + smtTRANSACTION, transHash, std::ref (getApp().getFullBelowCache()))) + , mAccountStateMap (boost::make_shared (smtSTATE, accountHash, + std::ref (getApp().getFullBelowCache()))) { updateHash (); loaded = true; @@ -140,7 +144,8 @@ Ledger::Ledger (bool /* dummy */, , mValidHash (false) , mAccepted (false) , mImmutable (false) - , mTransactionMap (boost::make_shared (smtTRANSACTION)) + , mTransactionMap (boost::make_shared (smtTRANSACTION, + std::ref (getApp().getFullBelowCache()))) , mAccountStateMap (prevLedger.mAccountStateMap->snapShot (true)) { prevLedger.updateHash (); @@ -181,8 +186,12 @@ Ledger::Ledger (Blob const& rawLedger, initializeFees (); } -Ledger::Ledger (const std::string& rawLedger, bool hasPrefix) : - mClosed (false), mValidated(false), mValidHash (false), mAccepted (false), mImmutable (true) +Ledger::Ledger (const std::string& rawLedger, bool hasPrefix) + : mClosed (false) + , mValidated (false) + , mValidHash (false) + , mAccepted (false) + , mImmutable (true) { Serializer s (rawLedger); setRaw (s, hasPrefix); @@ -261,8 +270,10 @@ void Ledger::setRaw (Serializer& s, bool hasPrefix) if (mValidHash) { - mTransactionMap = boost::make_shared (smtTRANSACTION, mTransHash); - mAccountStateMap = boost::make_shared (smtSTATE, mAccountHash); + mTransactionMap = boost::make_shared (smtTRANSACTION, mTransHash, + std::ref (getApp().getFullBelowCache())); + mAccountStateMap = boost::make_shared (smtSTATE, mAccountHash, + std::ref (getApp().getFullBelowCache())); } } diff --git a/src/ripple_app/ledger/LedgerHistory.cpp b/src/ripple_app/ledger/LedgerHistory.cpp index 674e54853..ca74fa281 100644 --- a/src/ripple_app/ledger/LedgerHistory.cpp +++ b/src/ripple_app/ledger/LedgerHistory.cpp @@ -31,13 +31,10 @@ LedgerHistory::LedgerHistory () : m_ledgers_by_hash ("LedgerCache", CACHED_LEDGER_NUM, CACHED_LEDGER_AGE, - get_abstract_clock (), - LogPartition::getJournal ()) + get_seconds_clock (), LogPartition::getJournal ()) , m_consensus_validated ("ConsensusValidated", 64, 300, - get_abstract_clock (), - LogPartition::getJournal ()) + get_seconds_clock (), LogPartition::getJournal ()) { - ; } void LedgerHistory::addLedger (Ledger::pointer ledger, bool validated) diff --git a/src/ripple_app/ledger/LedgerHistory.h b/src/ripple_app/ledger/LedgerHistory.h index 387f43c78..94cbc57d6 100644 --- a/src/ripple_app/ledger/LedgerHistory.h +++ b/src/ripple_app/ledger/LedgerHistory.h @@ -53,12 +53,12 @@ public: bool fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash); private: - typedef TaggedCacheType LedgersByHash; + typedef TaggedCache LedgersByHash; LedgersByHash m_ledgers_by_hash; //typedef std::pair - typedef TaggedCacheType > ConsensusValidated; ConsensusValidated m_consensus_validated; diff --git a/src/ripple_app/main/Application.cpp b/src/ripple_app/main/Application.cpp index 7a62a208e..009bdfa66 100644 --- a/src/ripple_app/main/Application.cpp +++ b/src/ripple_app/main/Application.cpp @@ -17,6 +17,11 @@ */ //============================================================================== +#include "../ripple/common/seconds_clock.h" +#include "Tuning.h" + +namespace ripple { + // VFALCO TODO Clean this global up static bool volatile doShutdown = false; @@ -107,6 +112,7 @@ public: std::unique_ptr m_collectorManager; std::unique_ptr m_resourceManager; std::unique_ptr m_rpcServiceManager; + std::unique_ptr m_fullBelowCache; // These are Stoppable-related NodeStoreScheduler m_nodeStoreScheduler; @@ -203,13 +209,11 @@ public: , m_nodeStoreManager (NodeStore::make_Manager ( std::move (make_Factories ()))) - , m_tempNodeCache ("NodeCache", 16384, 90, - get_abstract_clock (), - LogPartition::getJournal ()) + , m_tempNodeCache ("NodeCache", 16384, 90, get_seconds_clock (), + LogPartition::getJournal ()) - , m_sleCache ("LedgerEntryCache", 4096, 120, - get_abstract_clock (), - LogPartition::getJournal ()) + , m_sleCache ("LedgerEntryCache", 4096, 120, get_seconds_clock (), + LogPartition::getJournal ()) , m_collectorManager (CollectorManager::New ( getConfig().insightSettings, @@ -222,6 +226,10 @@ public: , m_rpcServiceManager (RPC::Manager::New ( LogPartition::getJournal ())) + , m_fullBelowCache (std::make_unique ( + "full_below", get_seconds_clock (), m_collectorManager->collector (), + fullBelowTargetSize, fullBelowExpirationSeconds)) + , m_nodeStoreScheduler (*this) // The JobQueue has to come pretty early since @@ -252,9 +260,8 @@ public: *m_jobQueue, LogPartition::getJournal ())) // VFALCO NOTE Does NetworkOPs depend on LedgerMaster? - , m_networkOPs (NetworkOPs::New (get_abstract_clock < - std::chrono::steady_clock, std::chrono::seconds> (), *m_ledgerMaster, - *m_jobQueue, LogPartition::getJournal ())) + , m_networkOPs (NetworkOPs::New (get_seconds_clock (), *m_ledgerMaster, + *m_jobQueue, LogPartition::getJournal ())) // VFALCO NOTE LocalCredentials starts the deprecated UNL service , m_deprecatedUNL (UniqueNodeList::New (*m_jobQueue)) @@ -271,8 +278,7 @@ public: , m_sntpClient (SNTPClient::New (*this)) - , m_inboundLedgers (InboundLedgers::New (get_abstract_clock < - std::chrono::steady_clock, std::chrono::seconds> (), *m_jobQueue)) + , m_inboundLedgers (InboundLedgers::New (get_seconds_clock (), *m_jobQueue)) , m_txQueue (TxQueue::New ()) @@ -337,6 +343,11 @@ public: return *m_rpcServiceManager; } + FullBelowCache& getFullBelowCache () + { + return *m_fullBelowCache; + } + JobQueue& getJobQueue () { return *m_jobQueue; @@ -1016,6 +1027,8 @@ public: // have listeners register for "onSweep ()" notification. // + m_fullBelowCache->sweep (); + logTimedCall (m_journal.warning, "TransactionMaster::sweep", __FILE__, __LINE__, boost::bind ( &TransactionMaster::sweep, &m_txMaster)); @@ -1395,4 +1408,4 @@ Application& getApp () return ApplicationImpBase::getInstance (); } -// class LoandObject (5removed, use git history to recover) +} diff --git a/src/ripple_app/main/Application.h b/src/ripple_app/main/Application.h index 2e2fd2039..fbd1c2c9b 100644 --- a/src/ripple_app/main/Application.h +++ b/src/ripple_app/main/Application.h @@ -20,6 +20,10 @@ #ifndef RIPPLE_APP_APPLICATION_H_INCLUDED #define RIPPLE_APP_APPLICATION_H_INCLUDED +#include "FullBelowCache.h" + +namespace ripple { + namespace SiteFiles { class Manager; } namespace Validators { class Manager; } namespace Resource { class Manager; } @@ -49,8 +53,8 @@ class PathRequests; class DatabaseCon; -typedef TaggedCacheType NodeCache; -typedef TaggedCacheType SLECache; +typedef TaggedCache NodeCache; +typedef TaggedCache SLECache; class Application : public PropertyStream::Source { @@ -79,6 +83,7 @@ public: virtual boost::asio::io_service& getIOService () = 0; virtual CollectorManager& getCollectorManager () = 0; virtual RPC::Manager& getRPCServiceManager() = 0; + virtual FullBelowCache& getFullBelowCache () = 0; virtual JobQueue& getJobQueue () = 0; virtual SiteFiles::Manager& getSiteFiles () = 0; virtual NodeCache& getTempNodeCache () = 0; @@ -147,4 +152,6 @@ std::unique_ptr make_Application(); // extern Application& getApp (); +} + #endif diff --git a/src/ripple_app/main/FullBelowCache.h b/src/ripple_app/main/FullBelowCache.h new file mode 100644 index 000000000..8b1f00bb8 --- /dev/null +++ b/src/ripple_app/main/FullBelowCache.h @@ -0,0 +1,32 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_FULLBELOWCACHE_H_INCLUDED +#define RIPPLE_FULLBELOWCACHE_H_INCLUDED + +#include "../ripple/types/api/UInt256.h" +#include "../ripple/radmap/api/BasicFullBelowCache.h" + +namespace ripple { + +typedef RadMap::BasicFullBelowCache FullBelowCache; + +} + +#endif diff --git a/src/ripple_app/main/Tuning.h b/src/ripple_app/main/Tuning.h new file mode 100644 index 000000000..632449103 --- /dev/null +++ b/src/ripple_app/main/Tuning.h @@ -0,0 +1,34 @@ +//------------------------------------------------------------------------------ +/* + This file is part of rippled: https://github.com/ripple/rippled + Copyright (c) 2012, 2013 Ripple Labs Inc. + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted, provided that the above + copyright notice and this permission notice appear in all copies. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL , DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +*/ +//============================================================================== + +#ifndef RIPPLE_APPLICATION_TUNING_H_INCLUDED +#define RIPPLE_APPLICATION_TUNING_H_INCLUDED + +namespace ripple { + +enum +{ + fullBelowTargetSize = 524288 + + ,fullBelowExpirationSeconds = 240 +}; + +} + +#endif diff --git a/src/ripple_app/misc/NetworkOPs.cpp b/src/ripple_app/misc/NetworkOPs.cpp index b8f7fc7d3..7bbff9d56 100644 --- a/src/ripple_app/misc/NetworkOPs.cpp +++ b/src/ripple_app/misc/NetworkOPs.cpp @@ -52,9 +52,8 @@ public: , mLastCloseConvergeTime (1000 * LEDGER_IDLE_INTERVAL) , mLastCloseTime (0) , mLastValidationTime (0) - , mFetchPack ("FetchPack", 65536, 45, - get_abstract_clock (), - LogPartition::getJournal ()) + , mFetchPack ("FetchPack", 65536, 45, clock, + LogPartition::getJournal ()) , mFetchSeq (0) , mLastLoadBase (256) , mLastLoadFactor (256) @@ -459,7 +458,7 @@ private: SubMapType mSubTransactions; // all accepted transactions SubMapType mSubRTTransactions; // all proposed and accepted transactions - TaggedCacheType< uint256, Blob> mFetchPack; + TaggedCache< uint256, Blob> mFetchPack; uint32 mFetchSeq; uint32 mLastLoadBase; diff --git a/src/ripple_app/misc/Validations.cpp b/src/ripple_app/misc/Validations.cpp index ee9914d61..c70ddfbab 100644 --- a/src/ripple_app/misc/Validations.cpp +++ b/src/ripple_app/misc/Validations.cpp @@ -32,7 +32,7 @@ private: typedef LockType::ScopedUnlockType ScopedUnlockType; LockType mLock; - TaggedCacheType mValidations; + TaggedCache mValidations; boost::unordered_map mCurrentValidations; std::vector mStaleValidations; @@ -60,9 +60,8 @@ private: public: ValidationsImp () : mLock (this, "Validations", __FILE__, __LINE__) - , mValidations ("Validations", 128, 600, - get_abstract_clock (), - LogPartition::getJournal ()) + , mValidations ("Validations", 128, 600, get_seconds_clock (), + LogPartition::getJournal ()) , mWriting (false) { mStaleValidations.reserve (512); diff --git a/src/ripple_app/ripple_app.cpp b/src/ripple_app/ripple_app.cpp index af82c49be..9c3f15c76 100644 --- a/src/ripple_app/ripple_app.cpp +++ b/src/ripple_app/ripple_app.cpp @@ -34,6 +34,7 @@ // This .cpp will end up including all of the public header // material in Ripple since it holds the Application object. +#include "../ripple/common/seconds_clock.h" #include "../ripple/http/ripple_http.h" #include "../ripple/resource/ripple_resource.h" #include "../ripple/rpc/ripple_rpc.h" @@ -47,12 +48,8 @@ # include "main/CollectorManager.h" #include "main/CollectorManager.cpp" + namespace ripple { - -// -// Application -// - # include "main/NodeStoreScheduler.h" #include "main/NodeStoreScheduler.cpp" @@ -90,15 +87,17 @@ namespace ripple { #include "websocket/WSConnection.cpp" # include "websocket/WSDoor.h" #include "websocket/WSDoor.cpp" +} + + #include "main/Application.cpp" -// -// RippleMain -// + + +namespace ripple { # include "main/RippleMain.h" #include "main/RippleMain.cpp" - } //------------------------------------------------------------------------------ diff --git a/src/ripple_app/ripple_app.h b/src/ripple_app/ripple_app.h index da8af341b..24a04a48f 100644 --- a/src/ripple_app/ripple_app.h +++ b/src/ripple_app/ripple_app.h @@ -66,6 +66,9 @@ // Order matters here. If you get compile errors, // reorder the include lines until the order is correct. +#include "../../ripple/common/KeyCache.h" +#include "../../ripple/common/TaggedCache.h" + namespace ripple { #include "data/Database.h" #include "data/DatabaseCon.h" @@ -123,7 +126,9 @@ namespace ripple { #include "misc/NetworkOPs.h" #include "tx/TransactionMaster.h" #include "main/LocalCredentials.h" +} // escape the namespace #include "main/Application.h" +namespace ripple { #include "ledger/OrderBookDB.h" #include "tx/Transactor.h" #include "tx/ChangeTransactor.h" diff --git a/src/ripple_app/ripple_app_pt2.cpp b/src/ripple_app/ripple_app_pt2.cpp index 4a95053cb..a8c4ca6ff 100644 --- a/src/ripple_app/ripple_app_pt2.cpp +++ b/src/ripple_app/ripple_app_pt2.cpp @@ -21,6 +21,8 @@ #include "ripple_app.h" +#include "../ripple/common/seconds_clock.h" + namespace ripple { #include "shamap/SHAMap.cpp" // Uses theApp diff --git a/src/ripple_app/ripple_app_pt4.cpp b/src/ripple_app/ripple_app_pt4.cpp index 8cf59526a..da4ab0451 100644 --- a/src/ripple_app/ripple_app_pt4.cpp +++ b/src/ripple_app/ripple_app_pt4.cpp @@ -23,6 +23,8 @@ #include "../ripple_net/ripple_net.h" +#include "../ripple/common/seconds_clock.h" + #include // for UniqueNodeList.cpp namespace ripple diff --git a/src/ripple_app/ripple_app_pt5.cpp b/src/ripple_app/ripple_app_pt5.cpp index a2e139cba..86a0dc129 100644 --- a/src/ripple_app/ripple_app_pt5.cpp +++ b/src/ripple_app/ripple_app_pt5.cpp @@ -22,6 +22,7 @@ #include "ripple_app.h" #include "../ripple_net/ripple_net.h" +#include "../ripple/common/seconds_clock.h" #include "../ripple/peerfinder/ripple_peerfinder.h" #include "../ripple/resource/ripple_resource.h" #include "../ripple/validators/ripple_validators.h" diff --git a/src/ripple_app/ripple_app_pt6.cpp b/src/ripple_app/ripple_app_pt6.cpp index f0d57e1d3..3ab780f41 100644 --- a/src/ripple_app/ripple_app_pt6.cpp +++ b/src/ripple_app/ripple_app_pt6.cpp @@ -21,8 +21,9 @@ #include "ripple_app.h" -namespace ripple -{ +#include "../ripple/common/seconds_clock.h" + +namespace ripple { #include "ledger/LedgerEntrySet.cpp" #include "ledger/AcceptedLedger.cpp" diff --git a/src/ripple_app/ripple_app_pt7.cpp b/src/ripple_app/ripple_app_pt7.cpp index aa13df0da..12057ae3d 100644 --- a/src/ripple_app/ripple_app_pt7.cpp +++ b/src/ripple_app/ripple_app_pt7.cpp @@ -23,6 +23,8 @@ #include "../ripple/resource/ripple_resource.h" +#include "../ripple/common/seconds_clock.h" + namespace ripple { diff --git a/src/ripple_app/rpc/RPCHandler.cpp b/src/ripple_app/rpc/RPCHandler.cpp index 24308c620..7b3e3d9c4 100644 --- a/src/ripple_app/rpc/RPCHandler.cpp +++ b/src/ripple_app/rpc/RPCHandler.cpp @@ -2925,7 +2925,7 @@ Json::Value RPCHandler::doGetCounts (Json::Value params, Resource::Charge& loadT ret["ledger_hit_rate"] = getApp().getLedgerMaster ().getCacheHitRate (); ret["AL_hit_rate"] = AcceptedLedger::getCacheHitRate (); - ret["fullbelow_size"] = SHAMap::getFullBelowSize (); + ret["fullbelow_size"] = int(getApp().getFullBelowCache().size()); ret["treenode_size"] = SHAMap::getTreeNodeSize (); std::string uptime; diff --git a/src/ripple_app/shamap/FetchPackTests.cpp b/src/ripple_app/shamap/FetchPackTests.cpp index 96ffd43f8..b243c6900 100644 --- a/src/ripple_app/shamap/FetchPackTests.cpp +++ b/src/ripple_app/shamap/FetchPackTests.cpp @@ -73,7 +73,11 @@ public: beginTestCase ("Build"); - boost::shared_ptr t1 (boost::make_shared
(smtFREE)); + FullBelowCache fullBelowCache ("test.full_below", + get_seconds_clock ()); + boost::shared_ptr
t1 (boost::make_shared
( + smtFREE, fullBelowCache)); + add_random_items (tableItems, *t1, random()); boost::shared_ptr
t2 (t1->snapShot (true)); @@ -93,7 +97,8 @@ public: { TestFilter filter (map, journal ()); - t3 = boost::make_shared
(smtFREE, t2->getHash () ); + t3 = boost::make_shared
(smtFREE, t2->getHash (), + fullBelowCache); expect (t3->fetchRoot (t2->getHash (), &filter), "unable to get root"); diff --git a/src/ripple_app/shamap/SHAMap.cpp b/src/ripple_app/shamap/SHAMap.cpp index e8d752b20..b6f273e3a 100644 --- a/src/ripple_app/shamap/SHAMap.cpp +++ b/src/ripple_app/shamap/SHAMap.cpp @@ -26,9 +26,10 @@ void SHAMap::DefaultMissingNodeHandler::operator() (uint32 refNUm) //------------------------------------------------------------------------------ -SHAMap::SHAMap (SHAMapType t, uint32 seq, +SHAMap::SHAMap (SHAMapType t, FullBelowCache& fullBelowCache, uint32 seq, MissingNodeHandler missing_node_handler) - : mSeq (seq) + : m_fullBelowCache (fullBelowCache) + , mSeq (seq) , mLedgerSeq (0) , mState (smsModifying) , mType (t) @@ -43,9 +44,10 @@ SHAMap::SHAMap (SHAMapType t, uint32 seq, mTNByID.replace(*root, root); } -SHAMap::SHAMap (SHAMapType t, uint256 const& hash, +SHAMap::SHAMap (SHAMapType t, uint256 const& hash, FullBelowCache& fullBelowCache, MissingNodeHandler missing_node_handler) - : mSeq (1) + : m_fullBelowCache (fullBelowCache) + , mSeq (1) , mLedgerSeq (0) , mState (smsSynching) , mType (t) @@ -59,9 +61,9 @@ SHAMap::SHAMap (SHAMapType t, uint256 const& hash, mTNByID.replace(*root, root); } -TaggedCacheType< SHAMap::TNIndex, SHAMapTreeNode> +TaggedCache< SHAMap::TNIndex, SHAMapTreeNode> SHAMap::treeNodeCache ("TreeNodeCache", 65536, 60, - get_abstract_clock (), + get_seconds_clock (), LogPartition::getJournal ()); SHAMap::~SHAMap () @@ -108,7 +110,8 @@ std::size_t hash_value (const SHAMapNode& mn) SHAMap::pointer SHAMap::snapShot (bool isMutable) { - SHAMap::pointer ret = boost::make_shared (mType); + SHAMap::pointer ret = boost::make_shared (mType, + std::ref (m_fullBelowCache)); SHAMap& newMap = *ret; // Return a new SHAMap that is a snapshot of this one @@ -1220,6 +1223,9 @@ public: { beginTestCase ("add/traverse"); + FullBelowCache fullBelowCache ("test.full_below", + get_seconds_clock ()); + // h3 and h4 differ only in the leaf, same terminal node (level 19) uint256 h1, h2, h3, h4, h5; h1.SetHex ("092891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); @@ -1228,7 +1234,7 @@ public: h4.SetHex ("b92891fe4ef6cee585fdc6fda2e09eb4d386363158ec3321b8123e5a772c6ca8"); h5.SetHex ("a92891fe4ef6cee585fdc6fda0e09eb4d386363158ec3321b8123e5a772c6ca7"); - SHAMap sMap (smtFREE); + SHAMap sMap (smtFREE, fullBelowCache); SHAMapItem i1 (h1, IntToVUC (1)), i2 (h2, IntToVUC (2)), i3 (h3, IntToVUC (3)), i4 (h4, IntToVUC (4)), i5 (h5, IntToVUC (5)); unexpected (!sMap.addItem (i2, true, false), "no add"); diff --git a/src/ripple_app/shamap/SHAMap.h b/src/ripple_app/shamap/SHAMap.h index 3188cc897..848a7c528 100644 --- a/src/ripple_app/shamap/SHAMap.h +++ b/src/ripple_app/shamap/SHAMap.h @@ -20,6 +20,9 @@ #ifndef RIPPLE_SHAMAP_H #define RIPPLE_SHAMAP_H +#include "../ripple/radmap/ripple_radmap.h" +#include "../main/FullBelowCache.h" + enum SHAMapState { smsModifying = 0, // Objects can be added and removed (like an open ledger) @@ -29,6 +32,8 @@ enum SHAMapState smsInvalid = 4, // Map is known not to be valid (usually synching a corrupt ledger) }; +//------------------------------------------------------------------------------ + namespace std { template <> @@ -42,6 +47,8 @@ struct hash } +//------------------------------------------------------------------------------ + namespace boost { template <> @@ -51,6 +58,8 @@ struct hash : std::hash } +//------------------------------------------------------------------------------ + namespace ripple { class SHAMap @@ -88,10 +97,10 @@ public: public: // build new map - explicit SHAMap (SHAMapType t, uint32 seq = 1, - MissingNodeHandler missing_node_handler = DefaultMissingNodeHandler()); + explicit SHAMap (SHAMapType t, FullBelowCache& fullBelowCache, + uint32 seq = 1, MissingNodeHandler missing_node_handler = DefaultMissingNodeHandler()); - SHAMap (SHAMapType t, uint256 const& hash, + SHAMap (SHAMapType t, uint256 const& hash, FullBelowCache& fullBelowCache, MissingNodeHandler missing_node_handler = DefaultMissingNodeHandler()); ~SHAMap (); @@ -245,19 +254,16 @@ public: static SHAMapTreeNode::pointer getCache (uint256 const& hash, SHAMapNode const& id); static void canonicalize (uint256 const& hash, SHAMapTreeNode::pointer&); - static int getFullBelowSize () - { - return fullBelowCache.size (); - } static int getTreeNodeSize () { return treeNodeCache.getCacheSize (); } + static void sweep () { - fullBelowCache.sweep (); treeNodeCache.sweep (); } + static void setTreeCache (int size, int age) { treeNodeCache.setTargetSize (size); @@ -267,9 +273,7 @@ public: typedef std::pair TNIndex; private: - static KeyCache fullBelowCache; - - static TaggedCacheType treeNodeCache; + static TaggedCache treeNodeCache; void dirtyUp (std::stack& stack, uint256 const & target, uint256 prevHash); std::stack getStack (uint256 const & id, bool include_nonmatching_leaf); @@ -307,6 +311,7 @@ private: // With a read lock, one may not invalidate pointers to existing members of mTNByID mutable LockType mLock; + FullBelowCache& m_fullBelowCache; uint32 mSeq; uint32 mLedgerSeq; // sequence number of ledger this is part of SyncUnorderedMapType< SHAMapNode, SHAMapTreeNode::pointer > mTNByID; diff --git a/src/ripple_app/shamap/SHAMapSync.cpp b/src/ripple_app/shamap/SHAMapSync.cpp index a466a365d..47254682d 100644 --- a/src/ripple_app/shamap/SHAMapSync.cpp +++ b/src/ripple_app/shamap/SHAMapSync.cpp @@ -21,12 +21,6 @@ static const uint256 uZero; -// VFALCO TODO Put these globals into a singleton and make the clock a parameter -KeyCache SHAMap::fullBelowCache ( - "fullBelowCache", - get_abstract_clock (), - 524288, 240); - void SHAMap::visitLeaves (std::function function) { // Make a snapshot of this map so we don't need to hold @@ -153,7 +147,7 @@ void SHAMap::getMissingNodes (std::vector& nodeIDs, std::vectorgetChildHash (branch); - if (! fullBelowCache.touch_if_exists (childHash)) + if (! m_fullBelowCache.touch_if_exists (childHash)) { SHAMapNode childID = node->getChildNodeID (branch); SHAMapTreeNode* d = getNodePointerNT (childID, childHash, filter); @@ -188,7 +182,7 @@ void SHAMap::getMissingNodes (std::vector& nodeIDs, std::vectorsetFullBelow (); if (mType == smtSTATE) - fullBelowCache.insert (node->getNodeHash ()); + m_fullBelowCache.insert (node->getNodeHash ()); } if (stack.empty ()) @@ -398,7 +392,7 @@ SHAMapAddNode SHAMap::addKnownNode (const SHAMapNode& node, Blob const& rawNode, return SHAMapAddNode::invalid (); } - if (fullBelowCache.touch_if_exists (iNode->getChildHash (branch))) + if (m_fullBelowCache.touch_if_exists (iNode->getChildHash (branch))) return SHAMapAddNode::duplicate (); SHAMapTreeNode *nextNode = getNodePointerNT (iNode->getChildNodeID (branch), iNode->getChildHash (branch), filter); @@ -752,14 +746,16 @@ public: RAND_pseudo_bytes (reinterpret_cast (&seed), sizeof (seed)); srand (seed); - SHAMap source (smtFREE), destination (smtFREE); + FullBelowCache fullBelowCache ("test.full_below", + get_seconds_clock ()); + + SHAMap source (smtFREE, fullBelowCache); + SHAMap destination (smtFREE, fullBelowCache); int items = 10000; for (int i = 0; i < items; ++i) source.addItem (*makeRandomAS (), false, false); - - beginTestCase ("add/remove"); unexpected (!confuseMap (source, 500), "ConfuseMap"); diff --git a/src/ripple_app/shamap/SHAMapSyncFilters.h b/src/ripple_app/shamap/SHAMapSyncFilters.h index 59d836b95..5ccd8975f 100644 --- a/src/ripple_app/shamap/SHAMapSyncFilters.h +++ b/src/ripple_app/shamap/SHAMapSyncFilters.h @@ -28,7 +28,7 @@ class ConsensusTransSetSF : public SHAMapSyncFilter { public: - typedef TaggedCacheType NodeCache; + typedef TaggedCache NodeCache; // VFALCO TODO Use a dependency injection to get the temp node cache ConsensusTransSetSF (NodeCache& nodeCache); diff --git a/src/ripple_app/tx/TransactionAcquire.cpp b/src/ripple_app/tx/TransactionAcquire.cpp index b013904ef..261e35022 100644 --- a/src/ripple_app/tx/TransactionAcquire.cpp +++ b/src/ripple_app/tx/TransactionAcquire.cpp @@ -35,7 +35,8 @@ TransactionAcquire::TransactionAcquire (uint256 const& hash, clock_type& clock) LogPartition::getJournal ()) , mHaveRoot (false) { - mMap = boost::make_shared (smtTRANSACTION, hash); + mMap = boost::make_shared (smtTRANSACTION, hash, + std::ref (getApp().getFullBelowCache ())); } TransactionAcquire::~TransactionAcquire () diff --git a/src/ripple_app/tx/TransactionMaster.cpp b/src/ripple_app/tx/TransactionMaster.cpp index 15a7c9c07..13e16c734 100644 --- a/src/ripple_app/tx/TransactionMaster.cpp +++ b/src/ripple_app/tx/TransactionMaster.cpp @@ -18,11 +18,9 @@ //============================================================================== TransactionMaster::TransactionMaster () - : mCache ("TransactionCache", 65536, 1800, - get_abstract_clock (), - LogPartition::getJournal ()) + : mCache ("TransactionCache", 65536, 1800, get_seconds_clock (), + LogPartition::getJournal ()) { - ; } bool TransactionMaster::inLedger (uint256 const& hash, uint32 ledger) diff --git a/src/ripple_app/tx/TransactionMaster.h b/src/ripple_app/tx/TransactionMaster.h index a8d5ead19..8cbb14e11 100644 --- a/src/ripple_app/tx/TransactionMaster.h +++ b/src/ripple_app/tx/TransactionMaster.h @@ -37,7 +37,7 @@ public: void sweep (void); private: - TaggedCacheType mCache; + TaggedCache mCache; }; #endif diff --git a/src/ripple_basics/ripple_basics.cpp b/src/ripple_basics/ripple_basics.cpp index d887dad2f..117a577b9 100644 --- a/src/ripple_basics/ripple_basics.cpp +++ b/src/ripple_basics/ripple_basics.cpp @@ -45,9 +45,7 @@ namespace ripple { -#include "containers/KeyCache.cpp" #include "containers/RangeSet.cpp" -#include "containers/TaggedCache.cpp" #include "log/Log.cpp" #include "log/LogFile.cpp" diff --git a/src/ripple_basics/ripple_basics.h b/src/ripple_basics/ripple_basics.h index a24a883be..e762e343d 100644 --- a/src/ripple_basics/ripple_basics.h +++ b/src/ripple_basics/ripple_basics.h @@ -85,7 +85,4 @@ using namespace beast; } -#include "containers/KeyCache.h" -#include "containers/TaggedCache.h" - #endif diff --git a/src/ripple_core/functional/JobQueue.cpp b/src/ripple_core/functional/JobQueue.cpp index bc4bf310a..1be4a6fcf 100644 --- a/src/ripple_core/functional/JobQueue.cpp +++ b/src/ripple_core/functional/JobQueue.cpp @@ -121,6 +121,8 @@ public: ~JobQueueImp () { + // Must unhook before destroying + m_metrics.hook = insight::Hook (); } void collect () diff --git a/src/ripple_core/nodestore/NodeStore.cpp b/src/ripple_core/nodestore/NodeStore.cpp index 05b4d722a..b10c366fa 100644 --- a/src/ripple_core/nodestore/NodeStore.cpp +++ b/src/ripple_core/nodestore/NodeStore.cpp @@ -27,6 +27,9 @@ #include "beast/beast/make_unique.h" +#include "../../ripple/common/seconds_clock.h" +#include "../../ripple/common/TaggedCache.h" + #include "impl/Tuning.h" # include "impl/DecodedBlob.h" # include "impl/EncodedBlob.h" diff --git a/src/ripple_core/nodestore/impl/DatabaseImp.h b/src/ripple_core/nodestore/impl/DatabaseImp.h index 8f1217551..ef1004f70 100644 --- a/src/ripple_core/nodestore/impl/DatabaseImp.h +++ b/src/ripple_core/nodestore/impl/DatabaseImp.h @@ -34,7 +34,7 @@ public: std::unique_ptr m_backend; // Larger key/value storage, but not necessarily persistent. std::unique_ptr m_fastBackend; - TaggedCacheType m_cache; + TaggedCache m_cache; DatabaseImp (std::string const& name, Scheduler& scheduler, @@ -46,8 +46,7 @@ public: , m_backend (std::move (backend)) , m_fastBackend (std::move (fastBackend)) , m_cache ("NodeStore", cacheTargetSize, cacheTargetSeconds, - get_abstract_clock (), - LogPartition::getJournal ()) + get_seconds_clock (), LogPartition::getJournal ()) { }