mirror of
https://github.com/XRPLF/rippled.git
synced 2026-08-21 06:10:58 +00:00
e41e469e02
Five sites copied the top entry out and then popped it. `SHAMapTreeNodePtr` is refcounted, so each copy bumped its atomic strong-ref count; `release()` moves the pointer out instead, a plain pointer swap with no atomic. `SHAMapNodeID` also derives from `CountedObject`, but `CountedObject` declares no move constructor, so moving a `SHAMapNodeID` still runs its copy constructor; `release()` saves nothing on the ID half of the pair. `dirtyUp` and `delItem` walk up to 64 levels per insert or delete on the ledger write path, so this removes up to 64 atomic operations per call, not 128. The two sites that read without popping now bind a reference rather than copying. `dirtyUp` and `delItem` both drop a `dynamicPointerCast`/null check for a static cast, on the reasoning that by the time either receives the stack, `addGiveItem`/`updateGiveItem` have already consumed the terminal leaf entry via `release()`, so every remaining entry is provably an inner node. That reasoning holds today, but an `XRPL_ASSERT` is a no-op under `NDEBUG`, so both get a real `UNREACHABLE`-guarded check instead: a release build that somehow violated the invariant would otherwise write through a misinterpreted node via `setChild`, silent memory corruption in place of the clean crash `dynamicPointerCast` used to produce. `updateGiveItem`'s own cast needed the same treatment for a different reason: an absent tag leaves an inner node on top of the stack, and the cast that followed the assertion there would have reinterpreted an `SHAMapInnerNode` as a `SHAMapLeafNode`. Replaced with `if (!top->isLeaf()) return false;`, pinned by a regression test. The `std::move` these sites previously applied to `staticPointerCast`/`dynamicPointerCast` was dropped rather than fixed: both only had a `TT const&` overload, so the move bound to that const ref and copied anyway, silently defeating the `SHAMapTreeNodePtr` refcount saving described above. Adds the missing rvalue overload to each, tied to `SharedIntrusive<TT>&&` rather than a bare `TT&&` so it cannot also bind to an lvalue in preference to the const-ref overload, and restores `std::move` at the three call sites that own a soon-to-be-discarded pointer.
559 lines
15 KiB
C++
559 lines
15 KiB
C++
#pragma once
|
|
|
|
#include <concepts>
|
|
#include <cstddef>
|
|
#include <cstdint>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
|
|
namespace xrpl {
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Tag to create an intrusive pointer from another intrusive pointer by using a
|
|
* static cast. This is useful to create an intrusive pointer to a derived
|
|
* class from an intrusive pointer to a base class.
|
|
*/
|
|
struct StaticCastTagSharedIntrusive
|
|
{
|
|
};
|
|
|
|
/**
|
|
* Tag to create an intrusive pointer from another intrusive pointer by using a
|
|
* dynamic cast. This is useful to create an intrusive pointer to a derived
|
|
* class from an intrusive pointer to a base class. If the cast fails an empty
|
|
* (null) intrusive pointer is created.
|
|
*/
|
|
struct DynamicCastTagSharedIntrusive
|
|
{
|
|
};
|
|
|
|
/**
|
|
* When creating or adopting a raw pointer, controls whether the strong count
|
|
* is incremented or not. Use this tag to increment the strong count.
|
|
*/
|
|
struct SharedIntrusiveAdoptIncrementStrongTag
|
|
{
|
|
};
|
|
|
|
/**
|
|
* When creating or adopting a raw pointer, controls whether the strong count
|
|
* is incremented or not. Use this tag to leave the strong count unchanged.
|
|
*/
|
|
struct SharedIntrusiveAdoptNoIncrementTag
|
|
{
|
|
};
|
|
|
|
//------------------------------------------------------------------------------
|
|
//
|
|
|
|
template <class T>
|
|
concept CAdoptTag = std::is_same_v<T, SharedIntrusiveAdoptIncrementStrongTag> ||
|
|
std::is_same_v<T, SharedIntrusiveAdoptNoIncrementTag>;
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* A shared intrusive pointer class that supports weak pointers.
|
|
*
|
|
* This is meant to be used for SHAMapInnerNodes, but may be useful for other
|
|
* cases. Since the reference counts are stored on the pointee, the pointee is
|
|
* not destroyed until both the strong _and_ weak pointer counts go to zero.
|
|
* When the strong pointer count goes to zero, the "partialDestructor" is
|
|
* called. This can be used to destroy as much of the object as possible while
|
|
* still retaining the reference counts. For example, for SHAMapInnerNodes the
|
|
* children may be reset in that function. Note that std::shared_pointer WILL
|
|
* run the destructor when the strong count reaches zero, but may not free the
|
|
* memory used by the object until the weak count reaches zero. In xrpld, we
|
|
* typically allocate shared pointers with the `make_shared` function. When
|
|
* that is used, the memory is not reclaimed until the weak count reaches zero.
|
|
*/
|
|
template <class T>
|
|
class SharedIntrusive
|
|
{
|
|
public:
|
|
SharedIntrusive() = default;
|
|
|
|
template <CAdoptTag TAdoptTag>
|
|
SharedIntrusive(T* p, TAdoptTag) noexcept;
|
|
|
|
SharedIntrusive(SharedIntrusive const& rhs);
|
|
|
|
template <class TT>
|
|
// TODO: convertible_to isn't quite right. That include a static castable.
|
|
// Find the right concept.
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedIntrusive(SharedIntrusive<TT> const& rhs);
|
|
|
|
SharedIntrusive(SharedIntrusive&& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedIntrusive(
|
|
SharedIntrusive<TT>&& rhs); // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
|
|
|
SharedIntrusive&
|
|
operator=(SharedIntrusive const& rhs);
|
|
|
|
bool
|
|
operator==(std::nullptr_t) const;
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedIntrusive&
|
|
operator=(SharedIntrusive<TT> const& rhs);
|
|
|
|
SharedIntrusive&
|
|
operator=(SharedIntrusive&& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedIntrusive&
|
|
operator=(
|
|
SharedIntrusive<TT>&& rhs); // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
|
|
|
/**
|
|
* Adopt the raw pointer. The strong reference may or may not be
|
|
* incremented, depending on the TAdoptTag
|
|
*/
|
|
template <CAdoptTag TAdoptTag = SharedIntrusiveAdoptIncrementStrongTag>
|
|
void
|
|
adopt(T* p);
|
|
|
|
~SharedIntrusive();
|
|
|
|
/**
|
|
* Create a new SharedIntrusive by statically casting the pointer
|
|
* controlled by the rhs param.
|
|
*/
|
|
template <class TT>
|
|
SharedIntrusive(StaticCastTagSharedIntrusive, SharedIntrusive<TT> const& rhs);
|
|
|
|
/**
|
|
* Create a new SharedIntrusive by statically casting the pointer
|
|
* controlled by the rhs param.
|
|
*/
|
|
template <class TT>
|
|
SharedIntrusive(StaticCastTagSharedIntrusive, SharedIntrusive<TT>&& rhs);
|
|
|
|
/**
|
|
* Create a new SharedIntrusive by dynamically casting the pointer
|
|
* controlled by the rhs param.
|
|
*/
|
|
template <class TT>
|
|
SharedIntrusive(DynamicCastTagSharedIntrusive, SharedIntrusive<TT> const& rhs);
|
|
|
|
/**
|
|
* Create a new SharedIntrusive by dynamically casting the pointer
|
|
* controlled by the rhs param.
|
|
*/
|
|
template <class TT>
|
|
SharedIntrusive(DynamicCastTagSharedIntrusive, SharedIntrusive<TT>&& rhs);
|
|
|
|
T&
|
|
operator*() const noexcept;
|
|
|
|
T*
|
|
operator->() const noexcept;
|
|
|
|
explicit
|
|
operator bool() const noexcept;
|
|
|
|
/**
|
|
* Set the pointer to null, decrement the strong count, and run the
|
|
* appropriate release action.
|
|
*/
|
|
void
|
|
reset();
|
|
|
|
/**
|
|
* Get the raw pointer
|
|
*/
|
|
[[nodiscard]] T*
|
|
get() const;
|
|
|
|
/**
|
|
* Return the strong count
|
|
*/
|
|
[[nodiscard]] std::size_t
|
|
useCount() const;
|
|
|
|
template <class TT, class... Args>
|
|
friend SharedIntrusive<TT>
|
|
makeSharedIntrusive(Args&&... args);
|
|
|
|
template <class TT>
|
|
friend class SharedIntrusive;
|
|
|
|
template <class TT>
|
|
friend class SharedWeakUnion;
|
|
|
|
template <class TT>
|
|
friend class WeakIntrusive;
|
|
|
|
private:
|
|
/**
|
|
* Return the raw pointer held by this object.
|
|
*/
|
|
[[nodiscard]] T*
|
|
unsafeGetRawPtr() const;
|
|
|
|
/**
|
|
* Exchange the current raw pointer held by this object with the given
|
|
* pointer. Decrement the strong count of the raw pointer previously held
|
|
* by this object and run the appropriate release action.
|
|
*/
|
|
void
|
|
unsafeReleaseAndStore(T* next);
|
|
|
|
/**
|
|
* Set the raw pointer directly. This is wrapped in a function so the class
|
|
* can support both atomic and non-atomic pointers in a future patch.
|
|
*/
|
|
void
|
|
unsafeSetRawPtr(T* p);
|
|
|
|
/**
|
|
* Exchange the raw pointer directly.
|
|
* This sets the raw pointer to the given value and returns the previous
|
|
* value. This is wrapped in a function so the class can support both
|
|
* atomic and non-atomic pointers in a future patch.
|
|
*/
|
|
T*
|
|
unsafeExchange(T* p);
|
|
|
|
/**
|
|
* pointer to the type with an intrusive count
|
|
*/
|
|
T* ptr_{nullptr};
|
|
};
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* A weak intrusive pointer class for the SharedIntrusive pointer class.
|
|
*
|
|
* Note that this weak pointer class asks differently from normal weak pointer
|
|
* classes. When the strong pointer count goes to zero, the "partialDestructor"
|
|
* is called. See the comment on SharedIntrusive for a fuller explanation.
|
|
*/
|
|
template <class T>
|
|
class WeakIntrusive
|
|
{
|
|
public:
|
|
WeakIntrusive() = default;
|
|
|
|
WeakIntrusive(WeakIntrusive const& rhs);
|
|
|
|
WeakIntrusive(WeakIntrusive&& rhs);
|
|
|
|
WeakIntrusive(SharedIntrusive<T> const& rhs);
|
|
|
|
// There is no move constructor from a strong intrusive ptr because
|
|
// moving would be move expensive than copying in this case (the strong
|
|
// ref would need to be decremented)
|
|
WeakIntrusive(SharedIntrusive<T> const&& rhs) = delete;
|
|
|
|
// Since there are no current use cases for copy assignment in
|
|
// WeakIntrusive, we delete this operator to simplify the implementation. If
|
|
// a need arises in the future, we can reintroduce it with proper
|
|
// consideration."
|
|
WeakIntrusive&
|
|
operator=(WeakIntrusive const&) = delete;
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
WeakIntrusive&
|
|
operator=(SharedIntrusive<TT> const& rhs);
|
|
|
|
/**
|
|
* Adopt the raw pointer and increment the weak count.
|
|
*/
|
|
void
|
|
adopt(T* ptr);
|
|
|
|
~WeakIntrusive();
|
|
|
|
/**
|
|
* Get a strong pointer from the weak pointer, if possible. This will
|
|
* only return a seated pointer if the strong count on the raw pointer
|
|
* is non-zero before locking.
|
|
*/
|
|
SharedIntrusive<T>
|
|
lock() const;
|
|
|
|
/**
|
|
* Return true if the strong count is zero.
|
|
*/
|
|
[[nodiscard]] bool
|
|
expired() const;
|
|
|
|
/**
|
|
* Set the pointer to null and decrement the weak count.
|
|
*
|
|
* Note: This may run the destructor if the strong count is zero.
|
|
*/
|
|
void
|
|
reset();
|
|
|
|
private:
|
|
T* ptr_ = nullptr;
|
|
|
|
/**
|
|
* Decrement the weak count. This does _not_ set the raw pointer to
|
|
* null.
|
|
*
|
|
* Note: This may run the destructor if the strong count is zero.
|
|
*/
|
|
void
|
|
unsafeReleaseNoStore();
|
|
};
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* A combination of a strong and a weak intrusive pointer stored in the
|
|
* space of a single pointer.
|
|
*
|
|
* This class is similar to a `std::variant<SharedIntrusive,WeakIntrusive>`
|
|
* with some optimizations. In particular, it uses a low-order bit to
|
|
* determine if the raw pointer represents a strong pointer or a weak
|
|
* pointer. It can also be quickly switched between its strong pointer and
|
|
* weak pointer representations. This class is useful for storing intrusive
|
|
* pointers in tagged caches.
|
|
*/
|
|
|
|
template <class T>
|
|
class SharedWeakUnion
|
|
{
|
|
// Tagged pointer. Low bit determines if this is a strong or a weak
|
|
// pointer. The low bit must be masked to zero when converting back to a
|
|
// pointer. If the low bit is '1', this is a weak pointer.
|
|
static_assert(alignof(T) >= 2, "Bad alignment: Combo pointer requires low bit to be zero");
|
|
|
|
public:
|
|
SharedWeakUnion() = default;
|
|
|
|
SharedWeakUnion(SharedWeakUnion const& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedWeakUnion(SharedIntrusive<TT> const& rhs);
|
|
|
|
SharedWeakUnion(SharedWeakUnion&& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedWeakUnion(
|
|
SharedIntrusive<TT>&& rhs); // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
|
|
|
SharedWeakUnion&
|
|
operator=(SharedWeakUnion const& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedWeakUnion&
|
|
operator=(SharedIntrusive<TT> const& rhs);
|
|
|
|
template <class TT>
|
|
requires std::convertible_to<TT*, T*>
|
|
SharedWeakUnion&
|
|
operator=(
|
|
SharedIntrusive<TT>&& rhs); // NOLINT(cppcoreguidelines-rvalue-reference-param-not-moved)
|
|
|
|
~SharedWeakUnion();
|
|
|
|
/**
|
|
* Return a strong pointer if this is already a strong pointer (i.e.
|
|
* don't lock the weak pointer. Use the `lock` method if that's what's
|
|
* needed)
|
|
*/
|
|
[[nodiscard]] SharedIntrusive<T>
|
|
getStrong() const;
|
|
|
|
/**
|
|
* Return true if this is a strong pointer and the strong pointer is
|
|
* seated.
|
|
*/
|
|
explicit
|
|
operator bool() const noexcept;
|
|
|
|
/**
|
|
* Set the pointer to null, decrement the appropriate ref count, and
|
|
* run the appropriate release action.
|
|
*/
|
|
void
|
|
reset();
|
|
|
|
/**
|
|
* If this is a strong pointer, return the raw pointer. Otherwise
|
|
* return null.
|
|
*/
|
|
[[nodiscard]] T*
|
|
get() const;
|
|
|
|
/**
|
|
* If this is a strong pointer, return the strong count. Otherwise
|
|
* return 0
|
|
*/
|
|
[[nodiscard]] std::size_t
|
|
useCount() const;
|
|
|
|
/**
|
|
* Return true if there is a non-zero strong count.
|
|
*/
|
|
[[nodiscard]] bool
|
|
expired() const;
|
|
|
|
/**
|
|
* If this is a strong pointer, return the strong pointer. Otherwise
|
|
* attempt to lock the weak pointer.
|
|
*/
|
|
[[nodiscard]] SharedIntrusive<T>
|
|
lock() const;
|
|
|
|
/**
|
|
* Return true is this represents a strong pointer.
|
|
*/
|
|
[[nodiscard]] bool
|
|
isStrong() const;
|
|
|
|
/**
|
|
* Return true is this represents a weak pointer.
|
|
*/
|
|
[[nodiscard]] bool
|
|
isWeak() const;
|
|
|
|
/**
|
|
* If this is a weak pointer, attempt to convert it to a strong
|
|
* pointer.
|
|
*
|
|
* @return true if successfully converted to a strong pointer (or was
|
|
* already a strong pointer). Otherwise false.
|
|
*/
|
|
bool
|
|
convertToStrong();
|
|
|
|
/**
|
|
* If this is a strong pointer, attempt to convert it to a weak
|
|
* pointer.
|
|
*
|
|
* @return false if the pointer is null. Otherwise return true.
|
|
*/
|
|
bool
|
|
convertToWeak();
|
|
|
|
private:
|
|
// Tagged pointer. Low bit determines if this is a strong or a weak
|
|
// pointer. The low bit must be masked to zero when converting back to a
|
|
// pointer. If the low bit is '1', this is a weak pointer.
|
|
std::uintptr_t tp_{0};
|
|
static constexpr std::uintptr_t kTagMask = 1;
|
|
static constexpr std::uintptr_t kPtrMask = ~kTagMask;
|
|
|
|
private:
|
|
/**
|
|
* Return the raw pointer held by this object.
|
|
*/
|
|
[[nodiscard]] T*
|
|
unsafeGetRawPtr() const;
|
|
|
|
enum class RefStrength { Strong, Weak };
|
|
/**
|
|
* Set the raw pointer and tag bit directly.
|
|
*/
|
|
void
|
|
unsafeSetRawPtr(T* p, RefStrength rs);
|
|
|
|
/**
|
|
* Set the raw pointer and tag bit to all zeros (strong null pointer).
|
|
*/
|
|
void unsafeSetRawPtr(std::nullptr_t);
|
|
|
|
/**
|
|
* Decrement the appropriate ref count, and run the appropriate release
|
|
* action. Note: this does _not_ set the raw pointer to null.
|
|
*/
|
|
void
|
|
unsafeReleaseNoStore();
|
|
};
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Create a shared intrusive pointer.
|
|
*
|
|
* Note: unlike std::shared_ptr, where there is an advantage of allocating
|
|
* the pointer and control block together, there is no benefit for intrusive
|
|
* pointers.
|
|
*/
|
|
template <class TT, class... Args>
|
|
SharedIntrusive<TT>
|
|
makeSharedIntrusive(Args&&... args)
|
|
{
|
|
auto p = new TT(std::forward<Args>(args)...);
|
|
|
|
static_assert(
|
|
noexcept(SharedIntrusive<TT>(
|
|
std::declval<TT*>(), std::declval<SharedIntrusiveAdoptNoIncrementTag>())),
|
|
"SharedIntrusive constructor should not throw or this can leak "
|
|
"memory");
|
|
|
|
return SharedIntrusive<TT>(p, SharedIntrusiveAdoptNoIncrementTag{});
|
|
}
|
|
|
|
//------------------------------------------------------------------------------
|
|
|
|
namespace intr_ptr {
|
|
template <class T>
|
|
using SharedPtr = SharedIntrusive<T>;
|
|
|
|
template <class T>
|
|
using WeakPtr = WeakIntrusive<T>;
|
|
|
|
template <class T>
|
|
using SharedWeakUnionPtr = SharedWeakUnion<T>;
|
|
|
|
template <class T, class... A>
|
|
SharedPtr<T>
|
|
makeShared(A&&... args)
|
|
{
|
|
return makeSharedIntrusive<T>(std::forward<A>(args)...);
|
|
}
|
|
|
|
template <class T, class TT>
|
|
SharedPtr<T>
|
|
staticPointerCast(TT const& v)
|
|
{
|
|
return SharedPtr<T>(StaticCastTagSharedIntrusive{}, v);
|
|
}
|
|
|
|
// A bare `TT&&` here would be a forwarding reference, since TT is deduced directly from this
|
|
// parameter, and would then also bind to lvalues in preference to the `const&` overload above
|
|
// (binding to a plain reference beats binding to a const one), silently moving out of a caller's
|
|
// live variable on what looks like a copy call. Naming the wrapped type keeps TT nested inside
|
|
// SharedIntrusive<TT>, so this only binds to an actual rvalue of that type.
|
|
template <class T, class TT>
|
|
SharedPtr<T>
|
|
staticPointerCast(SharedIntrusive<TT>&& v)
|
|
{
|
|
return SharedPtr<T>(StaticCastTagSharedIntrusive{}, std::move(v));
|
|
}
|
|
|
|
template <class T, class TT>
|
|
SharedPtr<T>
|
|
dynamicPointerCast(TT const& v)
|
|
{
|
|
return SharedPtr<T>(DynamicCastTagSharedIntrusive{}, v);
|
|
}
|
|
|
|
template <class T, class TT>
|
|
SharedPtr<T>
|
|
dynamicPointerCast(SharedIntrusive<TT>&& v)
|
|
{
|
|
return SharedPtr<T>(DynamicCastTagSharedIntrusive{}, std::move(v));
|
|
}
|
|
} // namespace intr_ptr
|
|
} // namespace xrpl
|