Merge branch 'ximinez/fix-getkeys' into ximinez/online-delete-gaps2

* ximinez/fix-getkeys:
  Experiment. Do not push
  Clean up the loop to make it more understandable, add logging
  fix: Add amendment sponsor for AccountRootsDeletedClean (7801)
  fix: Update base_uint and test changes released in 3.1.3 (7570)
  fix: Handle rounding just above kMaxRep more accurately (7389)
  fix: Document and assert "after" is never null in invariants (7354)
  Document the reasons for the post-allocation assert
  Add missed header
  Use the right type in include/xrpl/basics/TaggedCache.ipp
  Make the getKeys() allocation more robust
  Apply suggestion from @xrplf-ai-reviewer[bot]
  fix: Allocate TaggedCache::getKeys() memory outside of lock

include/xrpl/basics/Number.h

@@ -364,6 +364,8 @@ public:
     static constexpr internalrep kMaxRep = std::numeric_limits<rep>::max();
     static_assert(kMaxRep == 9'223'372'036'854'775'807);
     static_assert(-kMaxRep == std::numeric_limits<rep>::min() + 1);
+    static constexpr internalrep kMaxRepUp = ((kMaxRep / 10) + 1) * 10;
+    static_assert(kMaxRepUp == 9'223'372'036'854'775'810ULL);
 
     // May need to make unchecked private
     struct Unchecked
@@ -591,6 +593,13 @@ public:
     std::pair<T, int>
     normalizeToRange() const;
 
+    // Safely convert rep (int64) mantissa to internalrep (uint64). If the rep
+    // is negative, returns the positive value. This takes a little extra work
+    // because converting std::numeric_limits<std::int64_t>::min() flirts with
+    // UB, and can vary across compilers.
+    static internalrep
+    externalToInternal(rep mantissa);
+
 private:
     static thread_local RoundingMode mode;
     // The available ranges for mantissa
@@ -645,13 +654,6 @@ private:
     // exponent could go out of range, so it will be checked.
     [[nodiscard]] Number
     shiftExponent(int exponentDelta) const;
-
-    // Safely convert rep (int64) mantissa to internalrep (uint64). If the rep
-    // is negative, returns the positive value. This takes a little extra work
-    // because converting std::numeric_limits<std::int64_t>::min() flirts with
-    // UB, and can vary across compilers.
-    static internalrep
-    externalToInternal(rep mantissa);
 };
 
 constexpr Number::Number(bool negative, internalrep mantissa, int exponent, Unchecked) noexcept

include/xrpl/basics/TaggedCache.ipp

@@ -3,6 +3,9 @@
 #include <xrpl/basics/IntrusivePointer.ipp>
 #include <xrpl/basics/Log.h>  // IWYU pragma: keep
 #include <xrpl/basics/TaggedCache.h>
+#include <xrpl/basics/scope.h>
+
+#include <algorithm>
 
 namespace xrpl {
 
@@ -601,8 +604,43 @@ TaggedCache<Key, T, IsKeyCache, SharedWeakUnionPointer, SharedPointerType, Hash,
     std::vector<key_type> v;
 
     {
-        std::scoped_lock const lock(mutex_);
-        v.reserve(cache_.size());
+        // Keep track of how many iterations are needed. Exit the loop if the number of retries gets
+        // absurd. (Note that if this somehow ever happens, one more allocation will be done under
+        // lock, which is undesirable, but really should be almost impossible.)
+        std::size_t allocationIterations = 0;
+        std::unique_lock lock(mutex_);
+        for (auto size = cache_.size(); v.capacity() < size && allocationIterations < 20;
+             size = cache_.size())
+        {
+            ScopeUnlock const unlock(lock);
+            if (allocationIterations > 0)
+            {
+                JLOG(journal_.info())
+                    << "getKeys(): Cache grew beyond allocated capacity after "
+                    << allocationIterations << " prior attempt(s). Have " << v.capacity()
+                    << ", need " << size << ". Retrying allocation";
+            }
+            // Allocate the current size plus a little extra, in case the cache grows while
+            // allocating. Each time another allocation is needed, the extra also gets bigger until
+            // it ultimately doubles the size + 1.
+            constexpr std::size_t baseShift = 63;
+            auto const bufferOffset = std::min(allocationIterations, std::size_t{baseShift});
+            auto const bufferShift = baseShift - bufferOffset;
+            auto const buffer = (size >> bufferShift) + 1;
+            size += buffer;
+            v.reserve(size);
+            ++allocationIterations;
+        }
+        if (v.capacity() < cache_.size())
+        {
+            // LCOV_EXCL_START
+            UNREACHABLE("xrpl::TaggedCache::getKeys(): failed to allocate sufficient capacity");
+            v.reserve(cache_.size());
+            // LCOV_EXCL_STOP
+        }
+        XRPL_ASSERT(lock.owns_lock(), "xrpl::TaggedCache::getKeys(): owns lock");
+        XRPL_ASSERT(
+            v.capacity() >= cache_.size(), "xrpl::TaggedCache::getKeys(): sufficient capacity");
         for (auto const& _ : cache_)
             v.push_back(_.first);
     }

include/xrpl/basics/base_uint.h

@@ -308,7 +308,9 @@ public:
         XRPL_ASSERT(
             c.size() * sizeof(typename Container::value_type) == size(),
             "xrpl::BaseUInt::fromRaw(Container auto) : input size match");
-        std::memcpy(result.data_.data(), c.data(), size());
+        std::size_t const canCopy =
+            std::min(size(), c.size() * sizeof(typename Container::value_type));
+        std::memcpy(result.data_.data(), c.data(), canCopy);
         return result;
     }
 
@@ -322,7 +324,11 @@ public:
         XRPL_ASSERT(
             c.size() * sizeof(typename Container::value_type) == size(),
             "xrpl::BaseUInt::operator=(Container auto) : input size match");
-        std::memcpy(data_.data(), c.data(), size());
+        std::size_t const canCopy =
+            std::min(size(), c.size() * sizeof(typename Container::value_type));
+        if (canCopy < size())
+            *this = beast::kZero;
+        std::memcpy(data_.data(), c.data(), canCopy);
         return *this;
     }
 

include/xrpl/tx/invariants/InvariantCheck.h

@@ -71,9 +71,18 @@ public:
     /**
      * @brief called for each ledger entry in the current transaction.
      *
-     * @param isDelete true if the SLE is being deleted
-     * @param before ledger entry before modification by the transaction
-     * @param after ledger entry after modification by the transaction
+     * @param isDelete true if the SLE is being deleted.
+     * @param before ledger entry before modification by the transaction. `before` will be null if
+     *  the entry is new.
+     * @param after ledger entry after modification by the transaction. Always non-null. When
+     *  deleting, `after` may differ from `before`. Whether that is important is up to the
+     *  individual invariant check.
+     *
+     * @note `after` IS NEVER NULL. `isDelete` is the only correct way to check for deletions.
+     *  Do not make logic or branching decisions on whether on `after` is set, because it will
+     *  always be set. Treat a null `after` as a programming error (with XRPL_ASSERT). An
+     *  invariant MAY check for null defensively, if it makes more sense, but an assertion is
+     *  preferred for new invariants.
      */
     void
     visitEntry(bool isDelete, SLE::const_ref before, SLE::const_ref after);