mirror of
https://github.com/XRPLF/rippled.git
synced 2026-10-18 09:38:03 +00:00
Compare commits
3 Commits
nb-misc-cl
...
nb-spinloc
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
02e40e7a00 | ||
|
|
f14395427e | ||
|
|
b707bac74f |
@@ -1,29 +1,31 @@
|
||||
#pragma once
|
||||
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
|
||||
#include <boost/core/type_name.hpp>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <iterator>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/**
|
||||
* Manages all counted object types.
|
||||
*
|
||||
* Counters register themselves on a lock-free intrusive list maintained
|
||||
* by this object when constructed. Because counters are never destroyed
|
||||
* or removed, the ABA problem does not apply.
|
||||
*
|
||||
* The registry is iterable as a forward range.
|
||||
*/
|
||||
class CountedObjects
|
||||
{
|
||||
public:
|
||||
static CountedObjects&
|
||||
getInstance() noexcept;
|
||||
|
||||
using Entry = std::pair<std::string, int>;
|
||||
using List = std::vector<Entry>;
|
||||
|
||||
[[nodiscard]] List
|
||||
getCounts(int minimumThreshold) const;
|
||||
|
||||
public:
|
||||
/**
|
||||
* Implementation for @ref CountedObject.
|
||||
@@ -33,68 +35,147 @@ public:
|
||||
class Counter
|
||||
{
|
||||
public:
|
||||
Counter(std::string name) noexcept : name_(std::move(name)), count_(0)
|
||||
{
|
||||
// Insert ourselves at the front of the lock-free linked list
|
||||
CountedObjects& instance = CountedObjects::getInstance();
|
||||
Counter* head = nullptr;
|
||||
Counter(std::string name) noexcept;
|
||||
|
||||
do
|
||||
{
|
||||
head = instance.head_.load();
|
||||
next_ = head;
|
||||
} while (instance.head_.exchange(this) != head);
|
||||
// Counters are intrusive list nodes whose addresses are published
|
||||
// in the registry; they must never be copied or moved. The atomic
|
||||
// members already force this, but we make it explicit.
|
||||
Counter(Counter const&) = delete;
|
||||
Counter&
|
||||
operator=(Counter const&) = delete;
|
||||
Counter(Counter&&) = delete;
|
||||
Counter&
|
||||
operator=(Counter&&) = delete;
|
||||
|
||||
++instance.count_;
|
||||
}
|
||||
|
||||
~Counter() noexcept = default;
|
||||
|
||||
int
|
||||
std::uint32_t
|
||||
increment() noexcept
|
||||
{
|
||||
return ++count_;
|
||||
auto const newCount = count_.fetch_add(1, std::memory_order::relaxed) + 1;
|
||||
XRPL_ASSERT(newCount != 0, "xrpl::CountedObjects::Counter::increment : no overflow");
|
||||
|
||||
auto maxCount = maxCount_.load(std::memory_order::relaxed);
|
||||
|
||||
while (newCount > maxCount &&
|
||||
!maxCount_.compare_exchange_weak(maxCount, newCount, std::memory_order::relaxed))
|
||||
{
|
||||
}
|
||||
|
||||
return newCount;
|
||||
}
|
||||
|
||||
int
|
||||
std::uint32_t
|
||||
decrement() noexcept
|
||||
{
|
||||
return --count_;
|
||||
auto const prev = count_.fetch_sub(1, std::memory_order::relaxed);
|
||||
XRPL_ASSERT(prev != 0, "xrpl::CountedObjects::Counter::decrement : no underflow");
|
||||
return prev - 1;
|
||||
}
|
||||
|
||||
[[nodiscard]] int
|
||||
getCount() const noexcept
|
||||
[[nodiscard]] std::uint32_t
|
||||
count() const noexcept
|
||||
{
|
||||
return count_.load();
|
||||
return count_.load(std::memory_order::relaxed);
|
||||
}
|
||||
|
||||
[[nodiscard]] Counter*
|
||||
getNext() const noexcept
|
||||
[[nodiscard]] std::uint32_t
|
||||
max() const noexcept
|
||||
{
|
||||
return next_;
|
||||
return std::max(
|
||||
count_.load(std::memory_order::relaxed),
|
||||
maxCount_.load(std::memory_order::relaxed));
|
||||
}
|
||||
|
||||
[[nodiscard]] std::string const&
|
||||
getName() const noexcept
|
||||
name() const noexcept
|
||||
{
|
||||
return name_;
|
||||
}
|
||||
|
||||
private:
|
||||
friend class CountedObjects;
|
||||
|
||||
Counter* next_ = nullptr;
|
||||
std::atomic<std::uint32_t> count_ = 0;
|
||||
std::atomic<std::uint32_t> maxCount_ = 0;
|
||||
std::string const name_;
|
||||
std::atomic<int> count_;
|
||||
Counter* next_;
|
||||
};
|
||||
|
||||
private:
|
||||
CountedObjects() noexcept;
|
||||
~CountedObjects() noexcept = default;
|
||||
class Iterator
|
||||
{
|
||||
public:
|
||||
using value_type = Counter const;
|
||||
using reference = value_type&;
|
||||
using pointer = value_type*;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
|
||||
explicit Iterator(Counter* c = nullptr) noexcept : current_(c)
|
||||
{
|
||||
}
|
||||
|
||||
reference
|
||||
operator*() const noexcept
|
||||
{
|
||||
return *current_;
|
||||
}
|
||||
|
||||
pointer
|
||||
operator->() const noexcept
|
||||
{
|
||||
return current_;
|
||||
}
|
||||
|
||||
Iterator&
|
||||
operator++() noexcept
|
||||
{
|
||||
current_ = current_->next_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Iterator
|
||||
operator++(int) noexcept
|
||||
{
|
||||
auto tmp = *this;
|
||||
++*this;
|
||||
return tmp;
|
||||
}
|
||||
|
||||
bool
|
||||
operator==(Iterator const&) const noexcept = default;
|
||||
|
||||
private:
|
||||
Counter* current_;
|
||||
};
|
||||
|
||||
constexpr CountedObjects() noexcept = default;
|
||||
|
||||
[[nodiscard]] auto
|
||||
begin() const noexcept
|
||||
{
|
||||
return Iterator{head_.load(std::memory_order::acquire)};
|
||||
}
|
||||
|
||||
[[nodiscard]] auto
|
||||
end() const noexcept
|
||||
{
|
||||
return Iterator{};
|
||||
}
|
||||
|
||||
private:
|
||||
std::atomic<int> count_;
|
||||
std::atomic<Counter*> head_;
|
||||
std::atomic<Counter*> head_ = nullptr;
|
||||
};
|
||||
|
||||
/** The global counted object registry. */
|
||||
inline constinit CountedObjects gCountedObjects;
|
||||
|
||||
inline CountedObjects::Counter::Counter(std::string name) noexcept
|
||||
: next_(gCountedObjects.head_.load(std::memory_order::relaxed)), name_(std::move(name))
|
||||
{
|
||||
while (!gCountedObjects.head_.compare_exchange_weak(
|
||||
next_, this, std::memory_order::release, std::memory_order::relaxed))
|
||||
;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
@@ -103,27 +184,37 @@ private:
|
||||
* Derived classes have their instances counted automatically. This is used
|
||||
* for reporting purposes.
|
||||
*
|
||||
* The constructors are private and `Object` is befriended so that the
|
||||
* CRTP parameter must be the deriving class itself: a copy-paste error
|
||||
* like `class B : public CountedObject<A>` fails to compile instead of
|
||||
* silently polluting A's count.
|
||||
*
|
||||
* @note This class has no move operations by design: a derived class's
|
||||
* move constructor falls back to the copy constructor for this
|
||||
* base, so the newly created instance is counted. This keeps the
|
||||
* invariant that count is the number of outstanding subobjects.
|
||||
*
|
||||
* @warning Counted objects constructed during dynamic initialization of
|
||||
* other translation units may have their increments discarded when
|
||||
* counter itself is dynamically initialized. Do not create counted
|
||||
* objects before main() begins.
|
||||
*
|
||||
* @ingroup basics
|
||||
*/
|
||||
template <class Object>
|
||||
requires std::is_class_v<Object>
|
||||
class CountedObject
|
||||
{
|
||||
private:
|
||||
static auto&
|
||||
getCounter() noexcept
|
||||
{
|
||||
static CountedObjects::Counter kC{boost::core::type_name<Object>()};
|
||||
return kC;
|
||||
}
|
||||
static inline CountedObjects::Counter counter{boost::core::type_name<Object>()};
|
||||
|
||||
CountedObject() noexcept
|
||||
{
|
||||
getCounter().increment();
|
||||
counter.increment();
|
||||
}
|
||||
|
||||
CountedObject(CountedObject const&) noexcept
|
||||
{
|
||||
getCounter().increment();
|
||||
counter.increment();
|
||||
}
|
||||
|
||||
CountedObject&
|
||||
@@ -132,7 +223,7 @@ private:
|
||||
public:
|
||||
~CountedObject() noexcept
|
||||
{
|
||||
getCounter().decrement();
|
||||
counter.decrement();
|
||||
}
|
||||
|
||||
friend Object;
|
||||
|
||||
@@ -4,88 +4,215 @@
|
||||
|
||||
#include <xrpl/beast/utility/instrumentation.h>
|
||||
|
||||
#include <boost/predef/architecture.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <concepts>
|
||||
#include <limits>
|
||||
#include <type_traits>
|
||||
|
||||
#ifndef __aarch64__
|
||||
#if BOOST_ARCH_X86
|
||||
#include <immintrin.h>
|
||||
#endif
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
/** An unsigned integral type suitable for use as a spinlock.
|
||||
|
||||
The type must be always lock-free when wrapped in std::atomic, so
|
||||
that lock operations cannot themselves take a (library-level) lock.
|
||||
*/
|
||||
template <typename T>
|
||||
concept SpinlockValueType = std::is_unsigned_v<T> && std::atomic<T>::is_always_lock_free;
|
||||
|
||||
/** A spinlock value type that additionally supports the atomic bitwise
|
||||
operations required to pack multiple locks into a single integer.
|
||||
*/
|
||||
template <typename T>
|
||||
concept PackedSpinlockValueType = SpinlockValueType<T> && requires(std::atomic<T>& a, T v) {
|
||||
{ a.fetch_or(v) } -> std::same_as<T>;
|
||||
{ a.fetch_and(v) } -> std::same_as<T>;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
/**
|
||||
* Inform the processor that we are in a tight spin-wait loop.
|
||||
*
|
||||
* Spinlocks caught in tight loops can result in the processor's pipeline
|
||||
* filling up with comparison operations, resulting in a misprediction at
|
||||
* the time the lock is finally acquired, necessitating pipeline flushing
|
||||
* which is ridiculously expensive and results in very high latency.
|
||||
*
|
||||
* This function instructs the processor to "pause" for some architecture
|
||||
* specific amount of time, to prevent this.
|
||||
|
||||
/** Inform the processor that we are in a tight spin-wait loop.
|
||||
|
||||
Spinlocks caught in tight loops can result in the processor's pipeline
|
||||
filling up with comparison operations, resulting in a misprediction at
|
||||
the time the lock is finally acquired, necessitating pipeline flushing
|
||||
which is ridiculously expensive and results in very high latency.
|
||||
|
||||
This function instructs the processor to "pause" for some architecture
|
||||
specific amount of time, to prevent this.
|
||||
*/
|
||||
inline void
|
||||
spinPause() noexcept
|
||||
{
|
||||
#ifdef __aarch64__
|
||||
asm volatile("yield");
|
||||
#else
|
||||
#if BOOST_ARCH_X86
|
||||
_mm_pause();
|
||||
#elif BOOST_ARCH_ARM
|
||||
asm volatile("yield" ::: "memory");
|
||||
#else
|
||||
#error No implementation available for spinPause to use
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/** @{ */
|
||||
/**
|
||||
* Classes to handle arrays of spinlocks packed into a single atomic integer:
|
||||
*
|
||||
* Packed spinlocks allow for tremendously space-efficient lock-sharding
|
||||
* but they come at a cost.
|
||||
*
|
||||
* First, the implementation is necessarily low-level and uses advanced
|
||||
* features like memory ordering and highly platform-specific tricks to
|
||||
* maximize performance. This imposes a significant and ongoing cost to
|
||||
* developers.
|
||||
*
|
||||
* Second, and perhaps most important, is that the packing of multiple
|
||||
* locks into a single integer which, albeit space-efficient, also has
|
||||
* performance implications stemming from data dependencies, increased
|
||||
* cache-coherency traffic between processors and heavier loads on the
|
||||
* processor's load/store units.
|
||||
*
|
||||
* To be sure, these locks can have advantages but they are definitely
|
||||
* not general purpose locks and should not be thought of or used that
|
||||
* way. The use cases for them are likely few and far between; without
|
||||
* a compelling reason to use them, backed by profiling data, it might
|
||||
* be best to use one of the standard locking primitives instead. Note
|
||||
* that in most common platforms, `std::mutex` is so heavily optimized
|
||||
* that it can, usually, outperform spinlocks.
|
||||
*
|
||||
* @tparam T An unsigned integral type (e.g. std::uint16_t)
|
||||
*/
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* A class that grabs a single packed spinlock from an atomic integer.
|
||||
*
|
||||
* This class meets the requirements of Lockable:
|
||||
* https://en.cppreference.com/w/cpp/named_req/Lockable
|
||||
/** Attempt to acquire a spinlock without blocking.
|
||||
|
||||
@note This interface is primarily intended for one-shot attempts to
|
||||
acquire the lock. Avoid calling this function directly from a
|
||||
loop and use @ref spinLock instead.
|
||||
|
||||
@tparam T An unsigned integral type.
|
||||
@param lock The atomic variable used as the lock.
|
||||
@return true if the lock was acquired, false if it was already held.
|
||||
*/
|
||||
template <class T>
|
||||
template <SpinlockValueType T>
|
||||
[[nodiscard]] bool
|
||||
spinTryLock(std::atomic<T>& lock) noexcept
|
||||
{
|
||||
// A compare-exchange is required here, not an unconditional exchange:
|
||||
// a failed attempt must not modify the lock word, in case the atomic
|
||||
// is shared with PackedSpinlock).
|
||||
T expected = 0;
|
||||
|
||||
return lock.compare_exchange_strong(
|
||||
expected,
|
||||
std::numeric_limits<T>::max(),
|
||||
std::memory_order::acquire,
|
||||
std::memory_order::relaxed);
|
||||
;
|
||||
}
|
||||
|
||||
/** Acquire a spinlock, blocking until available.
|
||||
|
||||
Uses a TTAS (test-and-test-and-set) pattern, so waiters share the cache
|
||||
line read-only, helping to avoid unnecessary coherency traffic.
|
||||
|
||||
@tparam T An unsigned integral type.
|
||||
@param lock The atomic variable used as the lock.
|
||||
*/
|
||||
template <SpinlockValueType T>
|
||||
void
|
||||
spinLock(std::atomic<T>& lock) noexcept
|
||||
{
|
||||
do
|
||||
{
|
||||
// Relaxed ordering is sufficient for the spin: this load is only
|
||||
// a filter. The acquire on the successful exchange in spinTryLock
|
||||
// is what synchronizes the critical section.
|
||||
while (lock.load(std::memory_order::relaxed) != 0)
|
||||
detail::spinPause();
|
||||
} while (!spinTryLock(lock));
|
||||
}
|
||||
|
||||
/** Release a spinlock.
|
||||
|
||||
@tparam T An unsigned integral type.
|
||||
@param lock The atomic variable used as the lock.
|
||||
*/
|
||||
template <SpinlockValueType T>
|
||||
void
|
||||
spinUnlock(std::atomic<T>& lock) noexcept
|
||||
{
|
||||
lock.store(0, std::memory_order::release);
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/** A Lockable interface to a spinlock implemented on top of an atomic.
|
||||
|
||||
@tparam T An unsigned integral type.
|
||||
|
||||
@note Using `PackedSpinlock` and `Spinlock` against the same underlying
|
||||
atomic integer is possible but can result in `Spinlock` not being
|
||||
able to acquire the lock during periods of high contention due to
|
||||
the way the two locks operate: `Spinlock` spins and tries to grab
|
||||
all the bits at once, whereas any given `PackedSpinlock` instance
|
||||
only tries to grab one bit at a time. Caveat emptor.
|
||||
|
||||
This class meets the requirements of Lockable:
|
||||
https://en.cppreference.com/w/cpp/named_req/Lockable
|
||||
*/
|
||||
template <SpinlockValueType T>
|
||||
class Spinlock
|
||||
{
|
||||
std::atomic<T>& lock_;
|
||||
|
||||
public:
|
||||
Spinlock(Spinlock const&) = delete;
|
||||
Spinlock&
|
||||
operator=(Spinlock const&) = delete;
|
||||
|
||||
/** Construct a spinlock handle.
|
||||
|
||||
@param lock The atomic integer to spin against.
|
||||
|
||||
@note For performance reasons, you should strive to have `lock` be
|
||||
on a cacheline by itself.
|
||||
*/
|
||||
explicit Spinlock(std::atomic<T>& lock) noexcept : lock_(lock)
|
||||
{
|
||||
}
|
||||
|
||||
[[nodiscard]] bool
|
||||
try_lock() noexcept // NOLINT(readability-identifier-naming)
|
||||
{
|
||||
return spinTryLock(lock_);
|
||||
}
|
||||
|
||||
void
|
||||
lock() noexcept
|
||||
{
|
||||
spinLock(lock_);
|
||||
}
|
||||
|
||||
void
|
||||
unlock() noexcept
|
||||
{
|
||||
spinUnlock(lock_);
|
||||
}
|
||||
};
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
/** A Lockable interface to a packed spinlock implemented on top of an atomic.
|
||||
|
||||
Packed spinlocks offer tremendous space-efficient lock-sharding but
|
||||
they come at a cost.
|
||||
|
||||
First, the implementation is necessarily low-level and uses advanced
|
||||
features like memory ordering and highly platform-specific tricks to
|
||||
maximize performance. This imposes a significant and ongoing cost to
|
||||
developers.
|
||||
|
||||
Second, and perhaps most important, is that the packing of multiple
|
||||
locks into a single integer which, albeit space-efficient, also has
|
||||
performance implications stemming from data dependencies, increased
|
||||
cache-coherency traffic between processors and heavier loads on the
|
||||
processor's load/store units.
|
||||
|
||||
To be sure, these locks can have advantages but they are definitely
|
||||
not general purpose locks and should not be thought of or used that
|
||||
way. The use cases for them are likely few and far between; without
|
||||
a compelling reason to use them, backed by profiling data, it might
|
||||
be best to use one of the standard locking primitives instead. Note
|
||||
that in most common platforms, `std::mutex` is so heavily optimized
|
||||
that it can, usually, outperform spinlocks.
|
||||
|
||||
@tparam T An unsigned integral type (e.g. std::uint16_t)
|
||||
|
||||
This class meets the requirements of Lockable:
|
||||
https://en.cppreference.com/w/cpp/named_req/Lockable
|
||||
*/
|
||||
template <PackedSpinlockValueType T>
|
||||
class PackedSpinlock
|
||||
{
|
||||
// clang-format off
|
||||
static_assert(std::is_unsigned_v<T>);
|
||||
static_assert(std::atomic<T>::is_always_lock_free);
|
||||
static_assert(
|
||||
std::is_same_v<decltype(std::declval<std::atomic<T>&>().fetch_or(0)), T> &&
|
||||
std::is_same_v<decltype(std::declval<std::atomic<T>&>().fetch_and(0)), T>,
|
||||
"std::atomic<T>::fetch_and(T) and std::atomic<T>::fetch_and(T) are required by packed_spinlock");
|
||||
// clang-format on
|
||||
|
||||
private:
|
||||
std::atomic<T>& bits_;
|
||||
T const mask_;
|
||||
|
||||
@@ -94,120 +221,49 @@ public:
|
||||
PackedSpinlock&
|
||||
operator=(PackedSpinlock const&) = delete;
|
||||
|
||||
/**
|
||||
* A single spinlock packed inside the specified atomic
|
||||
*
|
||||
* @param lock The atomic integer inside which the spinlock is packed.
|
||||
* @param index The index of the spinlock this object acquires.
|
||||
*
|
||||
* @note For performance reasons, you should strive to have `lock` be
|
||||
* on a cacheline by itself.
|
||||
/** Construct a packed spinlock handle for a single bit.
|
||||
|
||||
@param lock The atomic integer inside which the spinlock is packed.
|
||||
@param index The index of the spinlock this object acquires.
|
||||
|
||||
@note For performance reasons, you should strive to have `lock` be
|
||||
on a cacheline by itself.
|
||||
*/
|
||||
PackedSpinlock(std::atomic<T>& lock, int index) : bits_(lock), mask_(static_cast<T>(1) << index)
|
||||
{
|
||||
XRPL_ASSERT(
|
||||
index >= 0 && (mask_ != 0),
|
||||
"xrpl::PackedSpinlock::PackedSpinlock : valid index and mask");
|
||||
}
|
||||
|
||||
[[nodiscard]] bool
|
||||
try_lock() // NOLINT(readability-identifier-naming)
|
||||
{
|
||||
return (bits_.fetch_or(mask_, std::memory_order_acquire) & mask_) == 0;
|
||||
}
|
||||
|
||||
void
|
||||
lock()
|
||||
{
|
||||
while (!try_lock())
|
||||
{
|
||||
// The use of relaxed memory ordering here is intentional and
|
||||
// serves to help reduce cache coherency traffic during times
|
||||
// of contention by avoiding writes that would definitely not
|
||||
// result in the lock being acquired.
|
||||
while ((bits_.load(std::memory_order_relaxed) & mask_) != 0)
|
||||
detail::spinPause();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
unlock()
|
||||
{
|
||||
bits_.fetch_and(~mask_, std::memory_order_release);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* A spinlock implemented on top of an atomic integer.
|
||||
*
|
||||
* @note Using `packed_spinlock` and `spinlock` against the same underlying
|
||||
* atomic integer can result in `spinlock` not being able to actually
|
||||
* acquire the lock during periods of high contention, because of how
|
||||
* the two locks operate: `spinlock` will spin trying to grab all the
|
||||
* bits at once, whereas any given `packed_spinlock` will only try to
|
||||
* grab one bit at a time. Caveat emptor.
|
||||
*
|
||||
* This class meets the requirements of Lockable:
|
||||
* https://en.cppreference.com/w/cpp/named_req/Lockable
|
||||
*/
|
||||
template <class T>
|
||||
class Spinlock
|
||||
{
|
||||
static_assert(std::is_unsigned_v<T>);
|
||||
static_assert(std::atomic<T>::is_always_lock_free);
|
||||
|
||||
private:
|
||||
std::atomic<T>& lock_;
|
||||
|
||||
public:
|
||||
Spinlock(Spinlock const&) = delete;
|
||||
Spinlock&
|
||||
operator=(Spinlock const&) = delete;
|
||||
|
||||
/**
|
||||
* Grabs the
|
||||
*
|
||||
* @param lock The atomic integer to spin against.
|
||||
*
|
||||
* @note For performance reasons, you should strive to have `lock` be
|
||||
* on a cacheline by itself.
|
||||
*/
|
||||
Spinlock(std::atomic<T>& lock) : lock_(lock)
|
||||
PackedSpinlock(std::atomic<T>& lock, int index) noexcept
|
||||
: bits_(lock), mask_([index]() {
|
||||
XRPL_ASSERT(
|
||||
index >= 0 && index < std::numeric_limits<T>::digits,
|
||||
"xrpl::PackedSpinlock::PackedSpinlock : valid index");
|
||||
return static_cast<T>(T{1} << index);
|
||||
}())
|
||||
{
|
||||
}
|
||||
|
||||
[[nodiscard]] bool
|
||||
try_lock() // NOLINT(readability-identifier-naming)
|
||||
try_lock() noexcept // NOLINT(readability-identifier-naming)
|
||||
{
|
||||
T expected = 0;
|
||||
|
||||
return lock_.compare_exchange_weak(
|
||||
expected,
|
||||
std::numeric_limits<T>::max(),
|
||||
std::memory_order_acquire,
|
||||
std::memory_order_relaxed);
|
||||
return (bits_.fetch_or(mask_, std::memory_order::acquire) & mask_) == 0;
|
||||
}
|
||||
|
||||
void
|
||||
lock()
|
||||
lock() noexcept
|
||||
{
|
||||
while (!try_lock())
|
||||
do
|
||||
{
|
||||
// The use of relaxed memory ordering here is intentional and
|
||||
// serves to help reduce cache coherency traffic during times
|
||||
// of contention by avoiding writes that would definitely not
|
||||
// result in the lock being acquired.
|
||||
while (lock_.load(std::memory_order_relaxed) != 0)
|
||||
// of contention by avoiding writes that are unlikely to grab
|
||||
// the requested lock.
|
||||
while ((bits_.load(std::memory_order::relaxed) & mask_) != 0)
|
||||
detail::spinPause();
|
||||
}
|
||||
} while (!try_lock());
|
||||
}
|
||||
|
||||
void
|
||||
unlock()
|
||||
unlock() noexcept
|
||||
{
|
||||
lock_.store(0, std::memory_order_release);
|
||||
bits_.fetch_and(~mask_, std::memory_order::release);
|
||||
}
|
||||
};
|
||||
/** @} */
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
@@ -101,22 +101,6 @@ operator<<(std::ostream& os, AccountID const& x)
|
||||
return os;
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the global cache used to map AccountID to base58 conversions.
|
||||
*
|
||||
* The cache is optional and need not be initialized. But because conversion
|
||||
* is expensive (it requires a SHA-256 operation) in most cases the overhead
|
||||
* of the cache is worth the benefit.
|
||||
*
|
||||
* @param count The number of entries the cache should accommodate. Zero will
|
||||
* disable the cache, releasing any memory associated with it.
|
||||
*
|
||||
* @note The function will only initialize the cache the first time it is
|
||||
* invoked. Subsequent invocations do nothing.
|
||||
*/
|
||||
void
|
||||
initAccountIdCache(std::size_t count);
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
||||
@@ -392,6 +392,7 @@ JSS(max_ledger); // in/out: LedgerCleaner
|
||||
JSS(max_queue_size); // out: TxQ
|
||||
JSS(max_spend_drops); // out: AccountInfo
|
||||
JSS(max_spend_drops_total); // out: AccountInfo
|
||||
JSS(maximum);
|
||||
JSS(mean); // out: get_aggregate_price
|
||||
JSS(median); // out: get_aggregate_price
|
||||
JSS(median_fee); // out: TxQ
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
#include <xrpl/basics/CountedObject.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
CountedObjects&
|
||||
CountedObjects::getInstance() noexcept
|
||||
{
|
||||
static CountedObjects kInstance;
|
||||
|
||||
return kInstance;
|
||||
}
|
||||
|
||||
CountedObjects::CountedObjects() noexcept : count_(0), head_(nullptr)
|
||||
{
|
||||
}
|
||||
|
||||
CountedObjects::List
|
||||
CountedObjects::getCounts(int minimumThreshold) const
|
||||
{
|
||||
List counts;
|
||||
|
||||
// When other operations are concurrent, the count
|
||||
// might be temporarily less than the actual count.
|
||||
counts.reserve(count_.load());
|
||||
|
||||
for (auto* ctr = head_.load(); ctr != nullptr; ctr = ctr->getNext())
|
||||
{
|
||||
if (ctr->getCount() >= minimumThreshold)
|
||||
counts.emplace_back(ctr->getName(), ctr->getCount());
|
||||
}
|
||||
|
||||
std::ranges::sort(counts);
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
} // namespace xrpl
|
||||
@@ -8,6 +8,8 @@
|
||||
#include <xrpl/protocol/digest.h>
|
||||
#include <xrpl/protocol/tokens.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cstdint>
|
||||
#include <cstring>
|
||||
@@ -19,85 +21,136 @@
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
namespace detail {
|
||||
namespace {
|
||||
|
||||
/**
|
||||
* Caches the base58 representations of AccountIDs
|
||||
/** If true, we implement a cache for account ID lookups.
|
||||
|
||||
This is generally a good idea and can improve performance, especially
|
||||
when serving client requests.
|
||||
|
||||
The cache may generate warnings under some sanitizers; if you want to
|
||||
disable caching, set this variable to false.
|
||||
*/
|
||||
class AccountIdCache
|
||||
constexpr bool cacheAccountIDs = true;
|
||||
|
||||
/** Represents a cached entry for the given AccountID.
|
||||
|
||||
We keep the size and alignment of the structure at 64 bytes to
|
||||
make sure it fits within a cache line and avoid false sharing.
|
||||
*/
|
||||
struct alignas(64) CachedAccountID
|
||||
{
|
||||
private:
|
||||
struct CachedAccountID
|
||||
{
|
||||
AccountID id;
|
||||
char encoding[40] = {0};
|
||||
};
|
||||
/** The sequence lock for the entry.
|
||||
|
||||
// The actual cache
|
||||
std::vector<CachedAccountID> cache_;
|
||||
An even value means that the data is stable, whereas an odd
|
||||
value indicates the data is being actively modified.
|
||||
|
||||
// We use a hash function designed to resist algorithmic complexity attacks
|
||||
HardenedHash<> hasher_;
|
||||
We use a 32-bit value to ensure that the counter will never
|
||||
return to an observed value during a read window.
|
||||
*/
|
||||
std::atomic<std::uint32_t> seq{0};
|
||||
|
||||
// 64 spinlocks, packed into a single 64-bit value
|
||||
std::atomic<std::uint64_t> locks_ = 0;
|
||||
/** The account id. */
|
||||
AccountID id{};
|
||||
|
||||
public:
|
||||
AccountIdCache(std::size_t count) : cache_(count)
|
||||
{
|
||||
// This is non-binding, but we try to avoid wasting memory that
|
||||
// is caused by overallocation.
|
||||
cache_.shrink_to_fit();
|
||||
}
|
||||
/** The length of the cached encoding.
|
||||
|
||||
std::string
|
||||
toBase58(AccountID const& id)
|
||||
{
|
||||
auto const index = hasher_(id) % cache_.size();
|
||||
This field disambiguates collisions between ACCOUNT_ZERO and
|
||||
never-written-to slots, since they would otherwise share the
|
||||
same encoded bits.
|
||||
*/
|
||||
std::uint8_t len = 0;
|
||||
|
||||
PackedSpinlock sl(locks_, index % 64);
|
||||
|
||||
{
|
||||
std::scoped_lock const lock(sl);
|
||||
|
||||
// The check against the first character of the encoding ensures
|
||||
// that we don't mishandle the case of the all-zero account:
|
||||
if (cache_[index].encoding[0] != 0 && cache_[index].id == id)
|
||||
return cache_[index].encoding;
|
||||
}
|
||||
|
||||
auto ret = encodeBase58Token(TokenType::AccountID, id.data(), id.size());
|
||||
|
||||
XRPL_ASSERT(ret.size() <= 38, "xrpl::detail::AccountIdCache : maximum result size");
|
||||
|
||||
{
|
||||
std::scoped_lock const lock(sl);
|
||||
cache_[index].id = id;
|
||||
std::strcpy(cache_[index].encoding, ret.c_str());
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
/** The actual string encoding. */
|
||||
char encoding[39] = {0};
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
static_assert(sizeof(CachedAccountID) == 64 && alignof(CachedAccountID) == 64);
|
||||
|
||||
static std::unique_ptr<detail::AccountIdCache> gAccountIdCache;
|
||||
// The actual cache to use: a simple, direct-mapped, best-effort construct.
|
||||
//
|
||||
// We do not use a hardened hash for this because an AccountID is already
|
||||
// just cryptographically generated (i.e. uniformly distributed) bits and
|
||||
// there is no amplification, since collisions do not carry an additional
|
||||
// cost.
|
||||
//
|
||||
// The size is a "nice" power of 2 because it makes the calculation of an
|
||||
// index easier.
|
||||
constinit std::array<CachedAccountID, 65536> gAccountIdCache{};
|
||||
|
||||
void
|
||||
initAccountIdCache(std::size_t count)
|
||||
{
|
||||
if (!gAccountIdCache && count != 0)
|
||||
gAccountIdCache = std::make_unique<detail::AccountIdCache>(count);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::string
|
||||
toBase58(AccountID const& v)
|
||||
{
|
||||
if (gAccountIdCache)
|
||||
return gAccountIdCache->toBase58(v);
|
||||
// Since the input bits are generated by a cryptographically secure
|
||||
// hash construct, we can just pick any subset of them, no muss, no
|
||||
// fuss.
|
||||
auto const index = [&v]() -> std::uint16_t {
|
||||
auto const data = v.data();
|
||||
|
||||
return encodeBase58Token(TokenType::AccountID, v.data(), v.size());
|
||||
return static_cast<std::uint16_t>(data[0]) | (static_cast<std::uint16_t>(data[1]) << 8);
|
||||
}();
|
||||
|
||||
if constexpr (cacheAccountIDs)
|
||||
{
|
||||
auto& slot = gAccountIdCache[index];
|
||||
|
||||
// This is the optimistic, store-free read path: a collision with a
|
||||
// concurrent writer will be detected because the seq recheck fails
|
||||
// and we fall through to the slow path.
|
||||
//
|
||||
// These unsynchronized reads of the slot fields are, formally, data
|
||||
// races and, technically speaking, result in undefined behavior and
|
||||
// a tool like TSAN might flag them. These should not have an impact
|
||||
// on the correctness of the returned data: the sequence lock checks
|
||||
// ensure we discard any data that was in flux.
|
||||
if (auto const s0 = slot.seq.load(std::memory_order::acquire);
|
||||
(s0 & 1) == 0 && slot.len != 0 && slot.id == v)
|
||||
{
|
||||
char buf[38];
|
||||
auto const len = std::min<std::size_t>(slot.len, sizeof(buf));
|
||||
std::memcpy(buf, slot.encoding, len);
|
||||
|
||||
// Prevents the loads above from sinking below the recheck; the
|
||||
// acquire on s0 constrains later operations, not earlier ones.
|
||||
std::atomic_thread_fence(std::memory_order::acquire);
|
||||
|
||||
// Everything read from the slot — the gates above and the bytes in
|
||||
// buf — precedes this recheck and is validated by it. No slot access
|
||||
// may follow it.
|
||||
if (slot.seq.load(std::memory_order::relaxed) == s0)
|
||||
return std::string(buf, len);
|
||||
}
|
||||
}
|
||||
|
||||
auto ret = encodeBase58Token(TokenType::AccountID, v.data(), v.size());
|
||||
|
||||
XRPL_ASSERT(!ret.empty() && ret.size() <= 38, "xrpl::toBase58 : maximum result size");
|
||||
|
||||
if constexpr (cacheAccountIDs)
|
||||
{
|
||||
auto& slot = gAccountIdCache[index];
|
||||
|
||||
// The CAS to an odd value excludes other writers; if it fails, another
|
||||
// thread wrote or is writing into this slot. That is not a problem: we
|
||||
// just don't store our result into the cache this time.
|
||||
if (auto s0 = slot.seq.load(std::memory_order::relaxed); (s0 & 1) == 0 &&
|
||||
slot.seq.compare_exchange_strong(s0, s0 + 1, std::memory_order::acquire))
|
||||
{
|
||||
slot.id = v;
|
||||
slot.len = static_cast<std::uint8_t>(ret.size());
|
||||
std::copy_n(ret.data(), ret.size(), slot.encoding);
|
||||
|
||||
// The acquire on the CAS above keeps the writes from becoming visible
|
||||
// before the seqlock transitions to an odd value, which means readers
|
||||
// can detect an in-progress write. This release publishes them before
|
||||
// the seqlock transitions to an even value.
|
||||
slot.seq.store(s0 + 2, std::memory_order::release);
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
template <>
|
||||
|
||||
@@ -305,12 +305,11 @@ struct Regression_test : public beast::unit_test::Suite
|
||||
return digest.asUInt256();
|
||||
}();
|
||||
|
||||
auto const mapCounts = [&](CountedObjects::List const& list) {
|
||||
std::map<std::string, int> result;
|
||||
for (auto const& e : list)
|
||||
{
|
||||
result[e.first] = e.second;
|
||||
}
|
||||
auto const mapCounts = [] {
|
||||
std::map<std::string, std::uint32_t> result;
|
||||
|
||||
for (auto const& c : gCountedObjects)
|
||||
result.emplace(c.name(), c.count());
|
||||
|
||||
return result;
|
||||
};
|
||||
@@ -319,11 +318,11 @@ struct Regression_test : public beast::unit_test::Suite
|
||||
{
|
||||
auto& cache = env.app().getCachedSLEs();
|
||||
cache.del(*digest, false); // NOLINT(bugprone-unchecked-optional-access)
|
||||
auto const beforeCounts = mapCounts(CountedObjects::getInstance().getCounts(0));
|
||||
auto const beforeCounts = mapCounts();
|
||||
|
||||
env(check::cash(alice, bobIndex, check::DeliverMin(XRP(100))), Ter(tecNO_ENTRY));
|
||||
|
||||
auto const afterCounts = mapCounts(CountedObjects::getInstance().getCounts(0));
|
||||
auto const afterCounts = mapCounts();
|
||||
|
||||
using namespace std::string_literals;
|
||||
BEAST_EXPECT(
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include <xrpl/json/json_value.h>
|
||||
#include <xrpl/protocol/jss.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
|
||||
namespace xrpl {
|
||||
@@ -52,36 +53,59 @@ class GetCounts_test : public beast::unit_test::Suite
|
||||
// check counts, default params
|
||||
result = env.rpc("get_counts")[jss::result];
|
||||
BEAST_EXPECT(result[jss::status] == "success");
|
||||
// compare with values reported by CountedObjects
|
||||
auto const& objectCounts = CountedObjects::getInstance().getCounts(10);
|
||||
for (auto const& it : objectCounts)
|
||||
|
||||
// compare with values reported by the registry
|
||||
auto const& counters = result[jss::counters];
|
||||
|
||||
for (auto const& c : gCountedObjects)
|
||||
{
|
||||
BEAST_EXPECTS(result.isMember(it.first), it.first);
|
||||
BEAST_EXPECTS(result[it.first].asInt() == it.second, it.first);
|
||||
if (auto const count = c.count(); count >= 10)
|
||||
{
|
||||
BEAST_EXPECTS(counters.isMember(c.name()), c.name());
|
||||
BEAST_EXPECTS(counters[c.name()][jss::current].asUInt() == count, c.name());
|
||||
}
|
||||
}
|
||||
|
||||
BEAST_EXPECT(!result.isMember(jss::local_txs));
|
||||
}
|
||||
|
||||
{
|
||||
// make request with min threshold 100 and verify
|
||||
// that only STObject and NodeObject are reported
|
||||
// that only counters at or above the threshold are reported
|
||||
result = env.rpc("get_counts", "100")[jss::result];
|
||||
BEAST_EXPECT(result[jss::status] == "success");
|
||||
|
||||
// compare with values reported by CountedObjects
|
||||
auto const& objectCounts = CountedObjects::getInstance().getCounts(100);
|
||||
for (auto const& it : objectCounts)
|
||||
{
|
||||
BEAST_EXPECTS(result.isMember(it.first), it.first);
|
||||
BEAST_EXPECTS(result[it.first].asInt() == it.second, it.first);
|
||||
}
|
||||
BEAST_EXPECT(!result.isMember("Transaction"));
|
||||
BEAST_EXPECT(!result.isMember("STTx"));
|
||||
BEAST_EXPECT(!result.isMember("STArray"));
|
||||
BEAST_EXPECT(!result.isMember("HashRouterEntry"));
|
||||
BEAST_EXPECT(!result.isMember("STLedgerEntry"));
|
||||
}
|
||||
auto const& counters = result[jss::counters];
|
||||
|
||||
// every registry counter at/above threshold must be reported, with
|
||||
// a matching current value
|
||||
for (auto const& c : gCountedObjects)
|
||||
{
|
||||
if (auto const count = c.count(); count >= 100)
|
||||
{
|
||||
BEAST_EXPECTS(counters.isMember(c.name()), c.name());
|
||||
BEAST_EXPECTS(counters[c.name()][jss::current].asUInt() == count, c.name());
|
||||
}
|
||||
}
|
||||
|
||||
// conversely, every reported entry must be a known counter that met
|
||||
// the threshold, and its maximum must be consistent
|
||||
for (auto const& name : counters.getMemberNames())
|
||||
{
|
||||
auto const it = std::ranges::find_if(
|
||||
gCountedObjects, [&](auto const& c) { return c.name() == name; });
|
||||
BEAST_EXPECTS(it != gCountedObjects.end(), name);
|
||||
|
||||
auto const& entry = counters[name];
|
||||
BEAST_EXPECTS(entry[jss::current].asUInt() >= 100, name);
|
||||
BEAST_EXPECTS(entry[jss::maximum].asUInt() >= entry[jss::current].asUInt(), name);
|
||||
}
|
||||
|
||||
BEAST_EXPECT(!counters.isMember("xrpl::Transaction"));
|
||||
BEAST_EXPECT(!counters.isMember("xrpl::STTx"));
|
||||
BEAST_EXPECT(!counters.isMember("xrpl::STArray"));
|
||||
BEAST_EXPECT(!counters.isMember("xrpl::HashRouterEntry"));
|
||||
BEAST_EXPECT(!counters.isMember("xrpl::STLedgerEntry"));
|
||||
}
|
||||
{
|
||||
// local_txs field will exist when there are open Txs
|
||||
env(pay(alice, bob, alice["USD"](5)));
|
||||
|
||||
@@ -473,8 +473,6 @@ public:
|
||||
getIoContext())
|
||||
, grpcServer_(std::make_unique<GRPCServer>(*this))
|
||||
{
|
||||
initAccountIdCache(config_->getValueFor(SizedItem::AccountIdCacheSize));
|
||||
|
||||
add(resourceManager_.get());
|
||||
|
||||
//
|
||||
|
||||
@@ -41,7 +41,6 @@ enum class SizedItem : std::size_t {
|
||||
OpenFinalLimit,
|
||||
BurstSize,
|
||||
RamSizeGb,
|
||||
AccountIdCacheSize,
|
||||
};
|
||||
|
||||
/**
|
||||
|
||||
@@ -111,7 +111,7 @@ namespace xrpl {
|
||||
|
||||
// clang-format off
|
||||
// The configurable node sizes are "tiny", "small", "medium", "large", "huge"
|
||||
inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 13>
|
||||
inline constexpr std::array<std::pair<SizedItem, std::array<int, 5>>, 12>
|
||||
kSizedItems
|
||||
{{
|
||||
// FIXME: We should document each of these items, explaining exactly
|
||||
@@ -130,8 +130,7 @@ kSizedItems
|
||||
{SizedItem::LgrDbCache, {{ 4, 8, 16, 32, 128 }}},
|
||||
{SizedItem::OpenFinalLimit, {{ 8, 16, 32, 64, 128 }}},
|
||||
{SizedItem::BurstSize, {{ 4, 8, 16, 32, 48 }}},
|
||||
{SizedItem::RamSizeGb, {{ 6, 8, 12, 24, 0 }}},
|
||||
{SizedItem::AccountIdCacheSize, {{ 20047, 50053, 77081, 150061, 300007 }}}
|
||||
{SizedItem::RamSizeGb, {{ 6, 8, 12, 24, 0 }}}
|
||||
}};
|
||||
// clang-format on
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
#include <xrpld/app/rdb/backend/SQLiteDatabase.h>
|
||||
#include <xrpld/rpc/Context.h>
|
||||
|
||||
#include <xrpl/basics/CountedObject.h>
|
||||
#include <xrpl/basics/UptimeClock.h>
|
||||
#include <xrpl/json/json_forwards.h>
|
||||
#include <xrpl/json/json_value.h>
|
||||
@@ -11,9 +12,11 @@
|
||||
#include <xrpl/protocol/jss.h>
|
||||
#include <xrpl/server/NetworkOPs.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
@@ -43,16 +46,26 @@ textTime(
|
||||
}
|
||||
|
||||
json::Value
|
||||
getCountsJson(Application& app, int minObjectCount)
|
||||
getCountsJson(Application& app, std::uint32_t minObjectCount)
|
||||
{
|
||||
auto objectCounts = CountedObjects::getInstance().getCounts(minObjectCount);
|
||||
json::Value ret = json::ValueType::Object;
|
||||
|
||||
json::Value ret(json::ValueType::Object);
|
||||
ret[jss::counters] = [minObjectCount] {
|
||||
json::Value ctrs = json::ValueType::Object;
|
||||
|
||||
for (auto const& [k, v] : objectCounts)
|
||||
{
|
||||
ret[k] = v;
|
||||
}
|
||||
for (auto const& c : gCountedObjects)
|
||||
{
|
||||
if (auto const count = c.count(); count >= minObjectCount)
|
||||
{
|
||||
json::Value obj(json::ValueType::Object);
|
||||
obj[jss::current] = count;
|
||||
obj[jss::maximum] = std::max(count, c.max());
|
||||
ctrs[c.name()] = std::move(obj);
|
||||
}
|
||||
}
|
||||
|
||||
return ctrs;
|
||||
}();
|
||||
|
||||
if (app.config().useTxTables())
|
||||
{
|
||||
@@ -113,7 +126,7 @@ getCountsJson(Application& app, int minObjectCount)
|
||||
json::Value
|
||||
doGetCounts(rpc::JsonContext& context)
|
||||
{
|
||||
int minCount = 10;
|
||||
std::uint32_t minCount = 10;
|
||||
|
||||
if (context.params.isMember(jss::min_count))
|
||||
minCount = context.params[jss::min_count].asUInt();
|
||||
|
||||
@@ -4,9 +4,11 @@
|
||||
|
||||
#include <xrpl/json/json_value.h>
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace xrpl {
|
||||
|
||||
json::Value
|
||||
getCountsJson(Application& app, int minObjectCount);
|
||||
getCountsJson(Application& app, std::uint32_t minObjectCount);
|
||||
|
||||
} // namespace xrpl
|
||||
|
||||
Reference in New Issue
Block a user