#include // IWYU pragma: keep #include // IWYU pragma: keep #include #include #include #include #include #include #include // IWYU pragma: keep #include #include #include #include #include #include #include #include #include #include #include #include #include namespace xrpl::tests { /** Experimentally, we discovered that using std::barrier performs extremely poorly (~1 hour vs ~1 minute to run the test suite) in certain macOS environments. To unblock our macOS CI pipeline, we replaced std::barrier with a custom mutex-based barrier (Barrier) that significantly improves performance without compromising correctness. For future reference, if we ever consider reintroducing std::barrier, the following configuration is known to exhibit the problem: Model Name: Mac mini Model Identifier: Mac14,3 Model Number: Z16K000R4LL/A Chip: Apple M2 Total Number of Cores: 8 (4 performance and 4 efficiency) Memory: 24 GB System Firmware Version: 11881.41.5 OS Loader Version: 11881.1.1 Apple clang version 16.0.0 (clang-1600.0.26.3) Target: arm64-apple-darwin24.0.0 Thread model: posix */ struct Barrier { std::mutex mtx; std::condition_variable cv; int count; int const initial; Barrier(int n) : count(n), initial(n) { } void arriveAndWait() { std::unique_lock lock(mtx); if (--count == 0) { count = initial; cv.notify_all(); } else { cv.wait(lock, [&] { return count == initial; }); } } }; namespace { enum class TrackedState : std::uint8_t { Uninitialized, Alive, PartiallyDeletedStarted, PartiallyDeleted, DeletedStarted, Deleted }; class TIBase : public IntrusiveRefCounts { public: static constexpr std::size_t kMaxStates = 128; static std::array, kMaxStates> state; static std::atomic nextId; static TrackedState getState(int id) { assert(id < state.size()); return state[id].load(std::memory_order_acquire); } static void resetStates(bool resetCallback) { for (int i = 0; i < kMaxStates; ++i) { state[i].store(TrackedState::Uninitialized, std::memory_order_release); } nextId.store(0, std::memory_order_release); if (resetCallback) TIBase::tracingCallback = [](TrackedState, std::optional) {}; } struct ResetStatesGuard { bool resetCallback{false}; ResetStatesGuard(bool resetCallback) : resetCallback{resetCallback} { TIBase::resetStates(resetCallback); } ~ResetStatesGuard() { TIBase::resetStates(resetCallback); } }; TIBase() : id{checkoutID()} { assert(state.size() > id); state[id].store(TrackedState::Alive, std::memory_order_relaxed); } ~TIBase() override { using enum TrackedState; assert(state.size() > id); tracingCallback(state[id].load(std::memory_order_relaxed), DeletedStarted); assert(state.size() > id); // Use relaxed memory order to try to avoid atomic operations from // adding additional memory synchronizations that may hide threading // errors in the underlying shared pointer class. state[id].store(DeletedStarted, std::memory_order_relaxed); tracingCallback(DeletedStarted, Deleted); assert(state.size() > id); state[id].store(TrackedState::Deleted, std::memory_order_relaxed); tracingCallback(TrackedState::Deleted, std::nullopt); } void partialDestructor() const { using enum TrackedState; assert(state.size() > id); tracingCallback(state[id].load(std::memory_order_relaxed), PartiallyDeletedStarted); assert(state.size() > id); state[id].store(PartiallyDeletedStarted, std::memory_order_relaxed); tracingCallback(PartiallyDeletedStarted, PartiallyDeleted); assert(state.size() > id); state[id].store(PartiallyDeleted, std::memory_order_relaxed); tracingCallback(PartiallyDeleted, std::nullopt); } static std::function)> tracingCallback; int id; private: static int checkoutID() { return nextId.fetch_add(1, std::memory_order_acq_rel); } }; std::array, TIBase::kMaxStates> TIBase::state; std::atomic TIBase::nextId{0}; std::function)> TIBase::tracingCallback = [](TrackedState, std::optional) {}; } // namespace class IntrusiveShared_test : public beast::unit_test::Suite { public: void testBasics() { testcase("Basics"); { TIBase::ResetStatesGuard const rsg{true}; TIBase const b; BEAST_EXPECT(b.useCount() == 1); b.addWeakRef(); BEAST_EXPECT(b.useCount() == 1); auto s = b.releaseStrongRef(); BEAST_EXPECT(s == ReleaseStrongRefAction::PartialDestroy); BEAST_EXPECT(b.useCount() == 0); TIBase const* pb = &b; partialDestructorFinished(&pb); BEAST_EXPECT(!pb); auto w = b.releaseWeakRef(); BEAST_EXPECT(w == ReleaseWeakRefAction::Destroy); } std::vector> strong; std::vector> weak; { TIBase::ResetStatesGuard const rsg{true}; using enum TrackedState; auto b = makeSharedIntrusive(); auto id = b->id; BEAST_EXPECT(TIBase::getState(id) == Alive); BEAST_EXPECT(b->useCount() == 1); for (int i = 0; i < 10; ++i) { strong.push_back(b); } b.reset(); BEAST_EXPECT(TIBase::getState(id) == Alive); strong.resize(strong.size() - 1); BEAST_EXPECT(TIBase::getState(id) == Alive); strong.clear(); BEAST_EXPECT(TIBase::getState(id) == Deleted); b = makeSharedIntrusive(); id = b->id; BEAST_EXPECT(TIBase::getState(id) == Alive); BEAST_EXPECT(b->useCount() == 1); for (int i = 0; i < 10; ++i) { weak.emplace_back(b); BEAST_EXPECT(b->useCount() == 1); } BEAST_EXPECT(TIBase::getState(id) == Alive); weak.resize(weak.size() - 1); BEAST_EXPECT(TIBase::getState(id) == Alive); b.reset(); BEAST_EXPECT(TIBase::getState(id) == PartiallyDeleted); while (!weak.empty()) { weak.resize(weak.size() - 1); if (!weak.empty()) BEAST_EXPECT(TIBase::getState(id) == PartiallyDeleted); } BEAST_EXPECT(TIBase::getState(id) == Deleted); } { TIBase::ResetStatesGuard const rsg{true}; using enum TrackedState; auto b = makeSharedIntrusive(); auto id = b->id; BEAST_EXPECT(TIBase::getState(id) == Alive); WeakIntrusive w{b}; BEAST_EXPECT(TIBase::getState(id) == Alive); auto s = w.lock(); BEAST_EXPECT(s && s->useCount() == 2); b.reset(); BEAST_EXPECT(TIBase::getState(id) == Alive); BEAST_EXPECT(s && s->useCount() == 1); s.reset(); BEAST_EXPECT(TIBase::getState(id) == PartiallyDeleted); BEAST_EXPECT(w.expired()); s = w.lock(); // Cannot convert a weak pointer to a strong pointer if object is // already partially deleted BEAST_EXPECT(!s); w.reset(); BEAST_EXPECT(TIBase::getState(id) == Deleted); } { TIBase::ResetStatesGuard const rsg{true}; using enum TrackedState; using swu = SharedWeakUnion; swu b = makeSharedIntrusive(); BEAST_EXPECT(b.isStrong() && b.useCount() == 1); auto id = b.get()->id; BEAST_EXPECT(TIBase::getState(id) == Alive); swu w = b; BEAST_EXPECT(TIBase::getState(id) == Alive); BEAST_EXPECT(w.isStrong() && b.useCount() == 2); w.convertToWeak(); BEAST_EXPECT(w.isWeak() && b.useCount() == 1); swu s = w; BEAST_EXPECT(s.isWeak() && b.useCount() == 1); s.convertToStrong(); BEAST_EXPECT(s.isStrong() && b.useCount() == 2); b.reset(); BEAST_EXPECT(TIBase::getState(id) == Alive); BEAST_EXPECT(s.useCount() == 1); BEAST_EXPECT(!w.expired()); s.reset(); BEAST_EXPECT(TIBase::getState(id) == PartiallyDeleted); BEAST_EXPECT(w.expired()); w.convertToStrong(); // Cannot convert a weak pointer to a strong pointer if object is // already partially deleted BEAST_EXPECT(w.isWeak()); w.reset(); BEAST_EXPECT(TIBase::getState(id) == Deleted); } { // Testing SharedWeakUnion assignment operator TIBase::ResetStatesGuard const rsg{true}; auto strong1 = makeSharedIntrusive(); auto strong2 = makeSharedIntrusive(); auto id1 = strong1->id; auto id2 = strong2->id; BEAST_EXPECT(id1 != id2); SharedWeakUnion union1 = strong1; SharedWeakUnion union2 = strong2; BEAST_EXPECT(union1.isStrong()); BEAST_EXPECT(union2.isStrong()); BEAST_EXPECT(union1.get() == strong1.get()); BEAST_EXPECT(union2.get() == strong2.get()); // 1) Normal assignment: explicitly calls SharedWeakUnion assignment union1 = union2; BEAST_EXPECT(union1.isStrong()); BEAST_EXPECT(union2.isStrong()); BEAST_EXPECT(union1.get() == union2.get()); BEAST_EXPECT(TIBase::getState(id1) == TrackedState::Alive); BEAST_EXPECT(TIBase::getState(id2) == TrackedState::Alive); // 2) Test self-assignment BEAST_EXPECT(union1.isStrong()); BEAST_EXPECT(TIBase::getState(id1) == TrackedState::Alive); int const initialRefCount = strong1->useCount(); #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wself-assign-overloaded" union1 = union1; // Self-assignment #pragma clang diagnostic pop BEAST_EXPECT(union1.isStrong()); BEAST_EXPECT(TIBase::getState(id1) == TrackedState::Alive); BEAST_EXPECT(strong1->useCount() == initialRefCount); // 3) Test assignment from null union pointer union1 = SharedWeakUnion(); BEAST_EXPECT(union1.get() == nullptr); // 4) Test assignment to expired union pointer strong2.reset(); union2.reset(); union1 = union2; BEAST_EXPECT(union1.get() == nullptr); BEAST_EXPECT(TIBase::getState(id2) == TrackedState::Deleted); } } void testPartialDelete() { testcase("Partial Delete"); // This test creates two threads. One with a strong pointer and one // with a weak pointer. The strong pointer is reset while the weak // pointer still holds a reference, triggering a partial delete. // While the partial delete function runs (a sleep is inserted) the // weak pointer is reset. The destructor should wait to run until // after the partial delete function has completed running. using enum TrackedState; TIBase::ResetStatesGuard const rsg{true}; auto strong = makeSharedIntrusive(); WeakIntrusive weak{strong}; bool destructorRan = false; bool partialDeleteRan = false; std::latch partialDeleteStartedSyncPoint{2}; strong->tracingCallback = [&](TrackedState cur, std::optional next) { using enum TrackedState; if (next == DeletedStarted) { // strong goes out of scope while weak is still in scope // This checks that partialDelete has run to completion // before the destructor is called. A sleep is inserted // inside the partial delete to make sure the destructor is // given an opportunity to run during partial delete. BEAST_EXPECT(cur == PartiallyDeleted); } if (next == PartiallyDeletedStarted) { partialDeleteStartedSyncPoint.arrive_and_wait(); using namespace std::chrono_literals; // Sleep and let the weak pointer go out of scope, // potentially triggering a destructor while partial delete // is running. The test is to make sure that doesn't happen. std::this_thread::sleep_for(800ms); } if (next == PartiallyDeleted) { BEAST_EXPECT(!partialDeleteRan && !destructorRan); partialDeleteRan = true; } if (next == Deleted) { BEAST_EXPECT(!destructorRan); destructorRan = true; } }; std::thread t1{[&] { partialDeleteStartedSyncPoint.arrive_and_wait(); weak.reset(); // Trigger a full delete as soon as the partial // delete starts }}; std::thread t2{[&] { strong.reset(); // Trigger a partial delete }}; t1.join(); t2.join(); BEAST_EXPECT(destructorRan && partialDeleteRan); } void testDestructor() { testcase("Destructor"); // This test creates two threads. One with a strong pointer and one // with a weak pointer. The weak pointer is reset while the strong // pointer still holds a reference. Then the strong pointer is // reset. Only the destructor should run. The partial destructor // should not be called. Since the weak reset runs to completion // before the strong pointer is reset, threading doesn't add much to // this test, but there is no harm in keeping it. using enum TrackedState; TIBase::ResetStatesGuard const rsg{true}; auto strong = makeSharedIntrusive(); WeakIntrusive weak{strong}; bool destructorRan = false; bool partialDeleteRan = false; std::latch weakResetSyncPoint{2}; strong->tracingCallback = [&](TrackedState cur, std::optional next) { using enum TrackedState; if (next == PartiallyDeleted) { BEAST_EXPECT(!partialDeleteRan && !destructorRan); partialDeleteRan = true; } if (next == Deleted) { BEAST_EXPECT(!destructorRan); destructorRan = true; } }; std::thread t1{[&] { weak.reset(); weakResetSyncPoint.arrive_and_wait(); }}; std::thread t2{[&] { weakResetSyncPoint.arrive_and_wait(); strong.reset(); // Trigger a partial delete }}; t1.join(); t2.join(); BEAST_EXPECT(destructorRan && !partialDeleteRan); } void testMultithreadedClearMixedVariant() { testcase("Multithreaded Clear Mixed Variant"); // This test creates and destroys many strong and weak pointers in a // loop. There is a random mix of strong and weak pointers stored in // a vector (held as a variant). Both threads clear all the pointers // and check that the invariants hold. using enum TrackedState; TIBase::ResetStatesGuard const rsg{true}; std::atomic destructionState{0}; // returns destructorRan and partialDestructorRan (in that order) auto getDestructorState = [&]() -> std::pair { int const s = destructionState.load(std::memory_order_relaxed); return {(s & 1) != 0, (s & 2) != 0}; }; auto setDestructorRan = [&]() -> void { destructionState.fetch_or(1, std::memory_order_acq_rel); }; auto setPartialDeleteRan = [&]() -> void { destructionState.fetch_or(2, std::memory_order_acq_rel); }; auto tracingCallback = [&](TrackedState cur, std::optional next) { using enum TrackedState; auto [destructorRan, partialDeleteRan] = getDestructorState(); if (next == PartiallyDeleted) { BEAST_EXPECT(!partialDeleteRan && !destructorRan); setPartialDeleteRan(); } if (next == Deleted) { BEAST_EXPECT(!destructorRan); setDestructorRan(); } }; auto createVecOfPointers = [&](auto const& toClone, std::default_random_engine& eng) -> std::vector, WeakIntrusive>> { std::vector, WeakIntrusive>> result; std::uniform_int_distribution<> toCreateDist(4, 64); std::uniform_int_distribution<> isStrongDist(0, 1); auto numToCreate = toCreateDist(eng); result.reserve(numToCreate); for (int i = 0; i < numToCreate; ++i) { if (isStrongDist(eng)) { result.emplace_back(SharedIntrusive(toClone)); } else { result.emplace_back(WeakIntrusive(toClone)); } } return result; }; static constexpr int kLoopIters = 2 * 1024; static constexpr int kNumThreads = 16; std::vector> toClone; Barrier loopStartSyncPoint{kNumThreads}; Barrier postCreateToCloneSyncPoint{kNumThreads}; Barrier postCreateVecOfPointersSyncPoint{kNumThreads}; auto engines = [&]() -> std::vector { std::random_device rd; std::vector result; result.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) result.emplace_back(rd()); return result; }(); // cloneAndDestroy clones the strong pointer into a vector of mixed // strong and weak pointers and destroys them all at once. // threadId==0 is special. auto cloneAndDestroy = [&](int threadId) { for (int i = 0; i < kLoopIters; ++i) { // ------ Sync Point ------ loopStartSyncPoint.arriveAndWait(); // only thread 0 should reset the state std::optional rsg; if (threadId == 0) { // Thread 0 is the genesis thread. It creates the strong // pointers to be cloned by the other threads. This // thread will also check that the destructor ran and // clear the temporary variables. rsg.emplace(false); auto [destructorRan, partialDeleteRan] = getDestructorState(); BEAST_EXPECT(!i || destructorRan); destructionState.store(0, std::memory_order_release); toClone.clear(); toClone.resize(kNumThreads); auto strong = makeSharedIntrusive(); strong->tracingCallback = tracingCallback; std::ranges::fill(toClone, strong); } // ------ Sync Point ------ postCreateToCloneSyncPoint.arriveAndWait(); auto v = createVecOfPointers(toClone[threadId], engines[threadId]); toClone[threadId].reset(); // ------ Sync Point ------ postCreateVecOfPointersSyncPoint.arriveAndWait(); v.clear(); } }; std::vector threads; threads.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) { threads.emplace_back(cloneAndDestroy, i); } for (int i = 0; i < kNumThreads; ++i) { threads[i].join(); } } void testMultithreadedClearMixedUnion() { testcase("Multithreaded Clear Mixed Union"); // This test creates and destroys many SharedWeak pointers in a // loop. All the pointers start as strong and a loop randomly // convert them between strong and weak pointers. Both threads clear // all the pointers and check that the invariants hold. // // Note: This test also differs from the test above in that the pointers // randomly change from strong to weak and from weak to strong in a // loop. This can't be done in the variant test above because variant is // not thread safe while the SharedWeakUnion is thread safe. using enum TrackedState; TIBase::ResetStatesGuard const rsg{true}; std::atomic destructionState{0}; // returns destructorRan and partialDestructorRan (in that order) auto getDestructorState = [&]() -> std::pair { int const s = destructionState.load(std::memory_order_relaxed); return {(s & 1) != 0, (s & 2) != 0}; }; auto setDestructorRan = [&]() -> void { destructionState.fetch_or(1, std::memory_order_acq_rel); }; auto setPartialDeleteRan = [&]() -> void { destructionState.fetch_or(2, std::memory_order_acq_rel); }; auto tracingCallback = [&](TrackedState cur, std::optional next) { using enum TrackedState; auto [destructorRan, partialDeleteRan] = getDestructorState(); if (next == PartiallyDeleted) { BEAST_EXPECT(!partialDeleteRan && !destructorRan); setPartialDeleteRan(); } if (next == Deleted) { BEAST_EXPECT(!destructorRan); setDestructorRan(); } }; auto createVecOfPointers = [&](auto const& toClone, std::default_random_engine& eng) -> std::vector> { std::vector> result; std::uniform_int_distribution<> toCreateDist(4, 64); auto numToCreate = toCreateDist(eng); result.reserve(numToCreate); for (int i = 0; i < numToCreate; ++i) result.emplace_back(SharedIntrusive(toClone)); return result; }; static constexpr int kLoopIters = 2 * 1024; static constexpr int kFlipPointersLoopIters = 256; static constexpr int kNumThreads = 16; std::vector> toClone; Barrier loopStartSyncPoint{kNumThreads}; Barrier postCreateToCloneSyncPoint{kNumThreads}; Barrier postCreateVecOfPointersSyncPoint{kNumThreads}; Barrier postFlipPointersLoopSyncPoint{kNumThreads}; auto engines = [&]() -> std::vector { std::random_device rd; std::vector result; result.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) result.emplace_back(rd()); return result; }(); // cloneAndDestroy clones the strong pointer into a vector of // mixed strong and weak pointers, runs a loop that randomly // changes strong pointers to weak pointers, and destroys them // all at once. auto cloneAndDestroy = [&](int threadId) { for (int i = 0; i < kLoopIters; ++i) { // ------ Sync Point ------ loopStartSyncPoint.arriveAndWait(); // only thread 0 should reset the state std::optional rsg; if (threadId == 0) { // threadId 0 is the genesis thread. It creates the // strong point to be cloned by the other threads. This // thread will also check that the destructor ran and // clear the temporary variables. rsg.emplace(false); auto [destructorRan, partialDeleteRan] = getDestructorState(); BEAST_EXPECT(!i || destructorRan); destructionState.store(0, std::memory_order_release); toClone.clear(); toClone.resize(kNumThreads); auto strong = makeSharedIntrusive(); strong->tracingCallback = tracingCallback; std::ranges::fill(toClone, strong); } // ------ Sync Point ------ postCreateToCloneSyncPoint.arriveAndWait(); auto v = createVecOfPointers(toClone[threadId], engines[threadId]); toClone[threadId].reset(); // ------ Sync Point ------ postCreateVecOfPointersSyncPoint.arriveAndWait(); std::uniform_int_distribution<> isStrongDist(0, 1); for (int f = 0; f < kFlipPointersLoopIters; ++f) { for (auto& p : v) { if (isStrongDist(engines[threadId])) { p.convertToStrong(); } else { p.convertToWeak(); } } } // ------ Sync Point ------ postFlipPointersLoopSyncPoint.arriveAndWait(); v.clear(); } }; std::vector threads; threads.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) { threads.emplace_back(cloneAndDestroy, i); } for (int i = 0; i < kNumThreads; ++i) { threads[i].join(); } } void testMultithreadedLockingWeak() { testcase("Multithreaded Locking Weak"); // This test creates a single shared atomic pointer that multiple thread // create weak pointers from. The threads then lock the weak pointers. // Both threads clear all the pointers and check that the invariants // hold. using enum TrackedState; TIBase::ResetStatesGuard const rsg{true}; std::atomic destructionState{0}; // returns destructorRan and partialDestructorRan (in that order) auto getDestructorState = [&]() -> std::pair { int const s = destructionState.load(std::memory_order_relaxed); return {(s & 1) != 0, (s & 2) != 0}; }; auto setDestructorRan = [&]() -> void { destructionState.fetch_or(1, std::memory_order_acq_rel); }; auto setPartialDeleteRan = [&]() -> void { destructionState.fetch_or(2, std::memory_order_acq_rel); }; auto tracingCallback = [&](TrackedState cur, std::optional next) { using enum TrackedState; auto [destructorRan, partialDeleteRan] = getDestructorState(); if (next == PartiallyDeleted) { BEAST_EXPECT(!partialDeleteRan && !destructorRan); setPartialDeleteRan(); } if (next == Deleted) { BEAST_EXPECT(!destructorRan); setDestructorRan(); } }; static constexpr int kLoopIters = 2 * 1024; static constexpr int kLockWeakLoopIters = 256; static constexpr int kNumThreads = 16; std::vector> toLock; Barrier loopStartSyncPoint{kNumThreads}; Barrier postCreateToLockSyncPoint{kNumThreads}; Barrier postLockWeakLoopSyncPoint{kNumThreads}; // lockAndDestroy creates weak pointers from the strong pointer // and runs a loop that locks the weak pointer. At the end of the loop // all the pointers are destroyed all at once. auto lockAndDestroy = [&](int threadId) { for (int i = 0; i < kLoopIters; ++i) { // ------ Sync Point ------ loopStartSyncPoint.arriveAndWait(); // only thread 0 should reset the state std::optional rsg; if (threadId == 0) { // threadId 0 is the genesis thread. It creates the // strong point to be locked by the other threads. This // thread will also check that the destructor ran and // clear the temporary variables. rsg.emplace(false); auto [destructorRan, partialDeleteRan] = getDestructorState(); BEAST_EXPECT(!i || destructorRan); destructionState.store(0, std::memory_order_release); toLock.clear(); toLock.resize(kNumThreads); auto strong = makeSharedIntrusive(); strong->tracingCallback = tracingCallback; std::ranges::fill(toLock, strong); } // ------ Sync Point ------ postCreateToLockSyncPoint.arriveAndWait(); // Multiple threads all create a weak pointer from the same // strong pointer WeakIntrusive const weak{toLock[threadId]}; for (int wi = 0; wi < kLockWeakLoopIters; ++wi) { BEAST_EXPECT(!weak.expired()); auto strong = weak.lock(); BEAST_EXPECT(strong); } // ------ Sync Point ------ postLockWeakLoopSyncPoint.arriveAndWait(); toLock[threadId].reset(); } }; std::vector threads; threads.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) { threads.emplace_back(lockAndDestroy, i); } for (int i = 0; i < kNumThreads; ++i) { threads[i].join(); } } void run() override { testBasics(); testPartialDelete(); testDestructor(); testMultithreadedClearMixedVariant(); testMultithreadedClearMixedUnion(); testMultithreadedLockingWeak(); } }; // namespace tests BEAST_DEFINE_TESTSUITE(IntrusiveShared, basics, xrpl); } // namespace xrpl::tests