Merge branch 'develop' into ximinez/directory

* XRPLF/develop:
  Explicitly trim the heap after cache sweeps (6022)
  ci: Update prepare-runner action to fix macOS build environment (empty)
  Disable featureBatch and fixBatchInnerSigs amendments (6402)
  chore: Add nix development environment (6314)

.clang-tidy

@@ -14,7 +14,6 @@ Checks: "-*,
   bugprone-fold-init-type,
   bugprone-forward-declaration-namespace,
   bugprone-inaccurate-erase,
-  bugprone-inc-dec-in-conditions,
   bugprone-incorrect-enable-if,
   bugprone-incorrect-roundings,
   bugprone-infinite-loop,
@@ -65,10 +64,8 @@ Checks: "-*,
   bugprone-undefined-memory-manipulation,
   bugprone-undelegated-constructor,
   bugprone-unhandled-exception-at-new,
-  bugprone-unhandled-self-assignment,
   bugprone-unique-ptr-array-mismatch,
   bugprone-unsafe-functions,
-  bugprone-unused-raii,
   bugprone-unused-local-non-trivial-variable,
   bugprone-virtual-near-miss,
   cppcoreguidelines-no-suspend-with-lock,
@@ -98,10 +95,12 @@ Checks: "-*,
 # checks that have some issues that need to be resolved:
 #
 # bugprone-crtp-constructor-accessibility,
+# bugprone-inc-dec-in-conditions,
 # bugprone-move-forwarding-reference,
 # bugprone-switch-missing-default-case,
 # bugprone-unused-return-value,
 # bugprone-use-after-move,
+# bugprone-unhandled-self-assignment,
 # bugprone-unused-raii,
 #
 # cppcoreguidelines-misleading-capture-default-by-value,

.github/scripts/strategy-matrix/generate.py

@@ -55,7 +55,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
             #   fee to 500.
             # - Bookworm using GCC 15: Debug on linux/amd64, enable code
             #   coverage (which will be done below).
-            # - Bookworm using Clang 16: Debug on linux/amd64, enable voidstar.
+            # - Bookworm using Clang 16: Debug on linux/arm64, enable voidstar.
             # - Bookworm using Clang 17: Release on linux/amd64, set the
             #   reference fee to 1000.
             # - Bookworm using Clang 20: Debug on linux/amd64.
@@ -78,7 +78,7 @@ def generate_strategy_matrix(all: bool, config: Config) -> list:
                     if (
                         f"{os['compiler_name']}-{os['compiler_version']}" == "clang-16"
                         and build_type == "Debug"
-                        and architecture["platform"] == "linux/amd64"
+                        and architecture["platform"] == "linux/arm64"
                     ):
                         cmake_args = f"-Dvoidstar=ON {cmake_args}"
                         skip = False

CMakeLists.txt

@@ -131,6 +131,7 @@ if(coverage)
     include(XrplCov)
 endif()
 
+set(PROJECT_EXPORT_SET XrplExports)
 include(XrplCore)
 include(XrplInstall)
 include(XrplValidatorKeys)

cmake/XrplConfig.cmake

@@ -0,0 +1,60 @@
+include(CMakeFindDependencyMacro)
+# need to represent system dependencies of the lib here
+#[=========================================================[
+  Boost
+#]=========================================================]
+if(static OR APPLE OR MSVC)
+    set(Boost_USE_STATIC_LIBS ON)
+endif()
+set(Boost_USE_MULTITHREADED ON)
+if(static OR MSVC)
+    set(Boost_USE_STATIC_RUNTIME ON)
+else()
+    set(Boost_USE_STATIC_RUNTIME OFF)
+endif()
+find_dependency(
+    Boost
+    COMPONENTS
+    chrono
+    container
+    context
+    coroutine
+    date_time
+    filesystem
+    program_options
+    regex
+    system
+    thread
+)
+#[=========================================================[
+  OpenSSL
+#]=========================================================]
+if(NOT DEFINED OPENSSL_ROOT_DIR)
+    if(DEFINED ENV{OPENSSL_ROOT})
+        set(OPENSSL_ROOT_DIR $ENV{OPENSSL_ROOT})
+    elseif(APPLE)
+        find_program(homebrew brew)
+        if(homebrew)
+            execute_process(
+                COMMAND ${homebrew} --prefix openssl
+                OUTPUT_VARIABLE OPENSSL_ROOT_DIR
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+            )
+        endif()
+    endif()
+    file(TO_CMAKE_PATH "${OPENSSL_ROOT_DIR}" OPENSSL_ROOT_DIR)
+endif()
+
+if(static OR APPLE OR MSVC)
+    set(OPENSSL_USE_STATIC_LIBS ON)
+endif()
+set(OPENSSL_MSVC_STATIC_RT ON)
+find_dependency(OpenSSL REQUIRED)
+find_dependency(ZLIB)
+find_dependency(date)
+if(TARGET ZLIB::ZLIB)
+    set_target_properties(
+        OpenSSL::Crypto
+        PROPERTIES INTERFACE_LINK_LIBRARIES ZLIB::ZLIB
+    )
+endif()

cmake/XrplInstall.cmake

@@ -2,38 +2,100 @@
    install stuff
 #]===================================================================]
 
-include(GNUInstallDirs)
+include(create_symbolic_link)
 
-if(is_root_project AND TARGET xrpld)
-    install(
-        TARGETS xrpld
-        RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT runtime
-    )
-
-    install(
-        FILES "${CMAKE_CURRENT_SOURCE_DIR}/cfg/xrpld-example.cfg"
-        DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/xrpld"
-        RENAME xrpld.cfg
-        COMPONENT runtime
-    )
-
-    install(
-        FILES "${CMAKE_CURRENT_SOURCE_DIR}/cfg/validators-example.txt"
-        DESTINATION "${CMAKE_INSTALL_SYSCONFDIR}/xrpld"
-        RENAME validators.txt
-        COMPONENT runtime
-    )
+# If no suffix is defined for executables (e.g. Windows uses .exe but Linux
+# and macOS use none), then explicitly set it to the empty string.
+if(NOT DEFINED suffix)
+    set(suffix "")
 endif()
 
 install(
-    TARGETS xrpl.libpb xrpl.libxrpl
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT development
-    ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT development
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT development
+    TARGETS
+        common
+        opts
+        xrpl_boost
+        xrpl_libs
+        xrpl_syslibs
+        xrpl.imports.main
+        xrpl.libpb
+        xrpl.libxrpl
+        xrpl.libxrpl.basics
+        xrpl.libxrpl.beast
+        xrpl.libxrpl.conditions
+        xrpl.libxrpl.core
+        xrpl.libxrpl.crypto
+        xrpl.libxrpl.git
+        xrpl.libxrpl.json
+        xrpl.libxrpl.rdb
+        xrpl.libxrpl.ledger
+        xrpl.libxrpl.net
+        xrpl.libxrpl.nodestore
+        xrpl.libxrpl.protocol
+        xrpl.libxrpl.resource
+        xrpl.libxrpl.server
+        xrpl.libxrpl.shamap
+        xrpl.libxrpl.tx
+        antithesis-sdk-cpp
+    EXPORT XrplExports
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib
+    RUNTIME DESTINATION bin
+    INCLUDES DESTINATION include
 )
 
 install(
     DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/include/xrpl"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-    COMPONENT development
+)
+
+install(
+    EXPORT XrplExports
+    FILE XrplTargets.cmake
+    NAMESPACE Xrpl::
+    DESTINATION lib/cmake/xrpl
+)
+include(CMakePackageConfigHelpers)
+write_basic_package_version_file(
+    XrplConfigVersion.cmake
+    VERSION ${xrpld_version}
+    COMPATIBILITY SameMajorVersion
+)
+
+if(is_root_project AND TARGET xrpld)
+    install(TARGETS xrpld RUNTIME DESTINATION bin)
+    set_target_properties(xrpld PROPERTIES INSTALL_RPATH_USE_LINK_PATH ON)
+    # sample configs should not overwrite existing files
+    # install if-not-exists workaround as suggested by
+    # https://cmake.org/Bug/view.php?id=12646
+    install(
+        CODE
+            "
+    macro (copy_if_not_exists SRC DEST NEWNAME)
+      if (NOT EXISTS \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/\${DEST}/\${NEWNAME}\")
+        file (INSTALL FILE_PERMISSIONS OWNER_READ OWNER_WRITE DESTINATION \"\${CMAKE_INSTALL_PREFIX}/\${DEST}\" FILES \"\${SRC}\" RENAME \"\${NEWNAME}\")
+      else ()
+        message (\"-- Skipping : \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/\${DEST}/\${NEWNAME}\")
+      endif ()
+    endmacro()
+    copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/xrpld-example.cfg\" etc xrpld.cfg)
+    copy_if_not_exists(\"${CMAKE_CURRENT_SOURCE_DIR}/cfg/validators-example.txt\" etc validators.txt)
+  "
+    )
+    install(
+        CODE
+            "
+    set(CMAKE_MODULE_PATH \"${CMAKE_MODULE_PATH}\")
+    include(create_symbolic_link)
+    create_symbolic_link(xrpld${suffix} \
+       \$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/rippled${suffix})
+  "
+    )
+endif()
+
+install(
+    FILES
+        ${CMAKE_CURRENT_SOURCE_DIR}/cmake/XrplConfig.cmake
+        ${CMAKE_CURRENT_BINARY_DIR}/XrplConfigVersion.cmake
+    DESTINATION lib/cmake/xrpl
 )

cmake/XrplSanity.cmake

@@ -50,13 +50,6 @@ if(MSVC AND CMAKE_GENERATOR_PLATFORM STREQUAL "Win32")
     message(FATAL_ERROR "Visual Studio 32-bit build is not supported.")
 endif()
 
-if(voidstar AND NOT is_amd64)
-    message(
-        FATAL_ERROR
-        "The voidstar library only supported on amd64/x86_64. Detected archictecture was: ${CMAKE_SYSTEM_PROCESSOR}"
-    )
-endif()
-
 if(APPLE AND NOT HOMEBREW)
     find_program(HOMEBREW brew)
 endif()

cmake/XrplValidatorKeys.cmake

@@ -22,8 +22,5 @@ if(validator_keys)
         validator-keys
         PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}"
     )
-    install(
-        TARGETS validator-keys
-        RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT runtime
-    )
+    install(TARGETS validator-keys RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif()

include/xrpl/protocol/detail/features.macro

@@ -16,6 +16,7 @@
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
 
+XRPL_FEATURE(DefragDirectories,          Supported::no,  VoteBehavior::DefaultNo)
 XRPL_FIX   (PermissionedDomainInvariant, Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (ExpiredNFTokenOfferRemoval, Supported::yes, VoteBehavior::DefaultNo)
 XRPL_FIX    (BatchInnerSigs,             Supported::no, VoteBehavior::DefaultNo)

src/libxrpl/basics/base64.cpp

@@ -168,7 +168,7 @@ decode(void* dest, char const* src, std::size_t len)
             break;
         ++in;
         c4[i] = v;
-        if (++i; i == 4)
+        if (++i == 4)
         {
             c3[0] = (c4[0] << 2) + ((c4[1] & 0x30) >> 4);
             c3[1] = ((c4[1] & 0xf) << 4) + ((c4[2] & 0x3c) >> 2);

src/libxrpl/json/json_reader.cpp

@@ -729,18 +729,21 @@ Reader::decodeUnicodeCodePoint(Token& token, Location& current, Location end, un
 
         unsigned int surrogatePair;
 
-        if (*current != '\\' || *(current + 1) != 'u')
+        if (*(current++) == '\\' && *(current++) == 'u')
+        {
+            if (decodeUnicodeEscapeSequence(token, current, end, surrogatePair))
+            {
+                unicode = 0x10000 + ((unicode & 0x3FF) << 10) + (surrogatePair & 0x3FF);
+            }
+            else
+                return false;
+        }
+        else
             return addError(
-                "expecting another \\u token to begin the second half of a unicode surrogate pair",
+                "expecting another \\u token to begin the second half of a "
+                "unicode surrogate pair",
                 token,
                 current);
-
-        current += 2;  // skip two characters checked above
-
-        if (!decodeUnicodeEscapeSequence(token, current, end, surrogatePair))
-            return false;
-
-        unicode = 0x10000 + ((unicode & 0x3FF) << 10) + (surrogatePair & 0x3FF);
     }
 
     return true;

src/libxrpl/json/json_writer.cpp

@@ -319,7 +319,7 @@ StyledWriter::writeValue(Value const& value)
                     document_ += " : ";
                     writeValue(childValue);
 
-                    if (++it; it == members.end())
+                    if (++it == members.end())
                         break;
 
                     document_ += ",";

src/libxrpl/ledger/ApplyView.cpp

@@ -10,6 +10,14 @@ namespace xrpl {
 
 namespace directory {
 
+struct Gap
+{
+    uint64_t const page;
+    SLE::pointer node;
+    uint64_t const nextPage;
+    SLE::pointer next;
+};
+
 std::uint64_t
 createRoot(
     ApplyView& view,
@@ -111,7 +119,9 @@ insertPage(
     if (page == 0)
         return std::nullopt;
     if (!view.rules().enabled(fixDirectoryLimit) && page >= dirNodeMaxPages)  // Old pages limit
+    {
         return std::nullopt;
+    }
 
     // We are about to create a new node; we'll link it to
     // the chain first:
@@ -133,12 +143,8 @@ insertPage(
     // it's the default.
     if (page != 1)
         node->setFieldU64(sfIndexPrevious, page - 1);
-    XRPL_ASSERT_PARTS(!nextPage, "xrpl::directory::insertPage", "nextPage has default value");
-    /* Reserved for future use when directory pages may be inserted in
-     * between two other pages instead of only at the end of the chain.
     if (nextPage)
         node->setFieldU64(sfIndexNext, nextPage);
-    */
     describe(node);
     view.insert(node);
 
@@ -154,7 +160,7 @@ ApplyView::dirAdd(
     uint256 const& key,
     std::function<void(std::shared_ptr<SLE> const&)> const& describe)
 {
-    auto root = peek(directory);
+    auto const root = peek(directory);
 
     if (!root)
     {
@@ -164,6 +170,43 @@ ApplyView::dirAdd(
 
     auto [page, node, indexes] = directory::findPreviousPage(*this, directory, root);
 
+    if (rules().enabled(featureDefragDirectories))
+    {
+        // If there are more nodes than just the root, and there's no space in
+        // the last one, walk backwards to find one with space, or to find one
+        // missing.
+        std::optional<directory::Gap> gapPages;
+        while (page && indexes.size() >= dirNodeMaxEntries)
+        {
+            // Find a page with space, or a gap in pages.
+            auto [prevPage, prevNode, prevIndexes] =
+                directory::findPreviousPage(*this, directory, node);
+            if (!gapPages && prevPage != page - 1)
+                gapPages.emplace(prevPage, prevNode, page, node);
+            page = prevPage;
+            node = prevNode;
+            indexes = prevIndexes;
+        }
+        // We looped through all the pages back to the root.
+        if (!page)
+        {
+            // If we found a gap, use it.
+            if (gapPages)
+            {
+                return directory::insertPage(
+                    *this,
+                    gapPages->page,
+                    gapPages->node,
+                    gapPages->nextPage,
+                    gapPages->next,
+                    key,
+                    directory,
+                    describe);
+            }
+            std::tie(page, node, indexes) = directory::findPreviousPage(*this, directory, root);
+        }
+    }
+
     // If there's space, we use it:
     if (indexes.size() < dirNodeMaxEntries)
     {

src/libxrpl/ledger/BookDirs.cpp

@@ -74,10 +74,8 @@ BookDirs::const_iterator::operator++()
     XRPL_ASSERT(index_ != zero, "xrpl::BookDirs::const_iterator::operator++ : nonzero index");
     if (!cdirNext(*view_, cur_key_, sle_, entry_, index_))
     {
-        if (index_ == 0)
-            cur_key_ = view_->succ(++cur_key_, next_quality_).value_or(zero);
-
-        if (index_ != 0 || cur_key_ == zero)
+        if (index_ != 0 ||
+            (cur_key_ = view_->succ(++cur_key_, next_quality_).value_or(zero)) == zero)
         {
             cur_key_ = key_;
             entry_ = 0;
@@ -86,7 +84,9 @@ BookDirs::const_iterator::operator++()
         else if (!cdirFirst(*view_, cur_key_, sle_, entry_, index_))
         {
             // LCOV_EXCL_START
-            UNREACHABLE("xrpl::BookDirs::const_iterator::operator++ : directory is empty");
+            UNREACHABLE(
+                "xrpl::BookDirs::const_iterator::operator++ : directory is "
+                "empty");
             // LCOV_EXCL_STOP
         }
     }

src/libxrpl/protocol/STBase.cpp

@@ -23,9 +23,6 @@ STBase::STBase(SField const& n) : fName(&n)
 STBase&
 STBase::operator=(STBase const& t)
 {
-    if (this == &t)
-        return *this;
-
     if (!fName->isUseful())
         fName = t.fName;
     return *this;

src/libxrpl/resource/Consumer.cpp

@@ -39,9 +39,6 @@ Consumer::~Consumer()
 Consumer&
 Consumer::operator=(Consumer const& other)
 {
-    if (this == &other)
-        return *this;
-
     // remove old ref
     if (m_logic && m_entry)
         m_logic->release(*m_entry);

src/test/jtx/Env_test.cpp

@@ -39,9 +39,9 @@ public:
             a = std::move(b);
             Account c(std::move(a));
         }
-        Account("alice");                      // NOLINT(bugprone-unused-raii)
-        Account("alice", KeyType::secp256k1);  // NOLINT(bugprone-unused-raii)
-        Account("alice", KeyType::ed25519);    // NOLINT(bugprone-unused-raii)
+        Account("alice");
+        Account("alice", KeyType::secp256k1);
+        Account("alice", KeyType::ed25519);
         auto const gw = Account("gw");
         [](AccountID) {}(gw);
         auto const USD = gw["USD"];
@@ -56,11 +56,11 @@ public:
     {
         using namespace jtx;
 
-        PrettyAmount(0);   // NOLINT(bugprone-unused-raii)
-        PrettyAmount(1);   // NOLINT(bugprone-unused-raii)
-        PrettyAmount(0u);  // NOLINT(bugprone-unused-raii)
-        PrettyAmount(1u);  // NOLINT(bugprone-unused-raii)
-        PrettyAmount(-1);  // NOLINT(bugprone-unused-raii)
+        PrettyAmount(0);
+        PrettyAmount(1);
+        PrettyAmount(0u);
+        PrettyAmount(1u);
+        PrettyAmount(-1);
         static_assert(!std::is_trivially_constructible<PrettyAmount, char>::value, "");
         static_assert(!std::is_trivially_constructible<PrettyAmount, unsigned char>::value, "");
         static_assert(!std::is_trivially_constructible<PrettyAmount, short>::value, "");