tagged cache reversion

Replace temporary uint256 with static variable when returning fallback hash
to avoid returning a const reference to a local temporary object.

.github/workflows/build-in-docker.yml

@@ -2,37 +2,104 @@ name: Build using Docker
 
 on:
   push:
-    branches: [ "dev", "candidate", "release", "jshooks" ]
+    branches: ["dev", "candidate", "release", "jshooks"]
   pull_request:
-    branches: [ "dev", "candidate", "release", "jshooks" ]
+    branches: ["dev", "candidate", "release", "jshooks"]
 
 concurrency:
-  group: ${{ github.workflow }}
-  cancel-in-progress: false
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP: 1
 
 jobs:
   checkout:
     runs-on: [self-hosted, vanity]
+    outputs:
+      checkout_path: ${{ steps.vars.outputs.checkout_path }}
     steps:
-    - uses: actions/checkout@v3
-      with:
-        clean: false
+      - name: Prepare checkout path
+        id: vars
+        run: |
+          SAFE_BRANCH=$(echo "${{ github.ref_name }}" | sed -e 's/[^a-zA-Z0-9._-]/-/g')
+          CHECKOUT_PATH="${SAFE_BRANCH}-${{ github.sha }}"
+          echo "checkout_path=${CHECKOUT_PATH}" >> "$GITHUB_OUTPUT"
+
+      - uses: actions/checkout@v4
+        with:
+          path: ${{ steps.vars.outputs.checkout_path }}
+          clean: true
+          fetch-depth: 2 # Only get the last 2 commits, to avoid fetching all history
+
   checkpatterns:
     runs-on: [self-hosted, vanity]
     needs: checkout
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Check for suspicious patterns
-      run: /bin/bash suspicious_patterns.sh
+      - name: Check for suspicious patterns
+        run: /bin/bash suspicious_patterns.sh
+
   build:
     runs-on: [self-hosted, vanity]
-    needs: checkpatterns
+    needs: [checkpatterns, checkout]
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Build using Docker
-      run: /bin/bash release-builder.sh
+      - name: Set Cleanup Script Path
+        run: |
+          echo "JOB_CLEANUP_SCRIPT=$(mktemp)" >> $GITHUB_ENV
+
+      - name: Build using Docker
+        run: /bin/bash release-builder.sh
+
+      - name: Stop Container (Cleanup)
+        if: always()
+        run: |
+          echo "Running cleanup script: $JOB_CLEANUP_SCRIPT"
+          /bin/bash -e -x "$JOB_CLEANUP_SCRIPT"
+          CLEANUP_EXIT_CODE=$?
+
+          if [[ "$CLEANUP_EXIT_CODE" -eq 0 ]]; then
+            echo "Cleanup script succeeded."
+            rm -f "$JOB_CLEANUP_SCRIPT"
+            echo "Cleanup script removed."
+          else
+            echo "⚠️ Cleanup script failed! Keeping for debugging: $JOB_CLEANUP_SCRIPT"
+          fi
+
+          if [[ "${DEBUG_BUILD_CONTAINERS_AFTER_CLEANUP}" == "1" ]]; then
+            echo "🔍 Checking for leftover containers..."
+            BUILD_CONTAINERS=$(docker ps --format '{{.Names}}' | grep '^xahaud_cached_builder' || echo "")
+
+            if [[ -n "$BUILD_CONTAINERS" ]]; then
+              echo "⚠️ WARNING: Some build containers are still running"
+              echo "$BUILD_CONTAINERS"
+            else
+              echo "✅ No build containers found"
+            fi
+          fi
+
   tests:
     runs-on: [self-hosted, vanity]
-    needs: build
+    needs: [build, checkout]
+    defaults:
+      run:
+        working-directory: ${{ needs.checkout.outputs.checkout_path }}
     steps:
-    - name: Unit tests
-      run: /bin/bash docker-unit-tests.sh
+      - name: Unit tests
+        run: /bin/bash docker-unit-tests.sh
 
+  cleanup:
+    runs-on: [self-hosted, vanity]
+    needs: [tests, checkout]
+    if: always()
+    steps:
+      - name: Cleanup workspace
+        run: |
+          CHECKOUT_PATH="${{ needs.checkout.outputs.checkout_path }}"
+          echo "Cleaning workspace for ${CHECKOUT_PATH}"
+          rm -rf "${{ github.workspace }}/${CHECKOUT_PATH}"

.github/workflows/clang-format.yml

@@ -30,7 +30,7 @@ jobs:
         git diff --exit-code | tee "clang-format.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: clang-format.patch

.github/workflows/levelization.yml

@@ -18,7 +18,7 @@ jobs:
         git diff --exit-code | tee "levelization.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: levelization.patch

.vscode/settings.json

@@ -3,7 +3,7 @@
   "C_Cpp.clang_format_path": ".clang-format",
   "C_Cpp.clang_format_fallbackStyle": "{ ColumnLimit: 0 }",
   "[cpp]":{
-    "editor.wordBasedSuggestions": false,
+    "editor.wordBasedSuggestions": "off",
     "editor.suggest.insertMode": "replace",
     "editor.semanticHighlighting.enabled": true,
     "editor.tabSize": 4,

Builds/CMake/RippledCore.cmake

@@ -392,6 +392,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/misc/NegativeUNLVote.cpp
   src/ripple/app/misc/NetworkOPs.cpp
   src/ripple/app/misc/SHAMapStoreImp.cpp
+  src/ripple/app/misc/StateAccounting.cpp
   src/ripple/app/misc/detail/impl/WorkSSL.cpp
   src/ripple/app/misc/impl/AccountTxPaging.cpp
   src/ripple/app/misc/impl/AmendmentTable.cpp
@@ -538,7 +539,9 @@ target_sources (rippled PRIVATE
        subdir: nodestore
   #]===============================]
   src/ripple/nodestore/backend/CassandraFactory.cpp
+  src/ripple/nodestore/backend/RWDBFactory.cpp
   src/ripple/nodestore/backend/MemoryFactory.cpp
+  src/ripple/nodestore/backend/FlatmapFactory.cpp
   src/ripple/nodestore/backend/NuDBFactory.cpp
   src/ripple/nodestore/backend/NullFactory.cpp
   src/ripple/nodestore/backend/RocksDBFactory.cpp
@@ -603,6 +606,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/BlackList.cpp
   src/ripple/rpc/handlers/BookOffers.cpp
   src/ripple/rpc/handlers/CanDelete.cpp
+  src/ripple/rpc/handlers/Catalogue.cpp
   src/ripple/rpc/handlers/Connect.cpp
   src/ripple/rpc/handlers/ConsensusInfo.cpp
   src/ripple/rpc/handlers/CrawlShards.cpp
@@ -658,6 +662,7 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/handlers/ValidatorListSites.cpp
   src/ripple/rpc/handlers/Validators.cpp
   src/ripple/rpc/handlers/WalletPropose.cpp
+  src/ripple/rpc/handlers/Catalogue.cpp
   src/ripple/rpc/impl/DeliveredAmount.cpp
   src/ripple/rpc/impl/Handler.cpp
   src/ripple/rpc/impl/LegacyPathFind.cpp
@@ -669,6 +674,9 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/impl/ShardVerificationScheduler.cpp
   src/ripple/rpc/impl/Status.cpp
   src/ripple/rpc/impl/TransactionSign.cpp
+  src/ripple/rpc/impl/NFTokenID.cpp
+  src/ripple/rpc/impl/NFTokenOfferID.cpp
+  src/ripple/rpc/impl/NFTSyntheticSerializer.cpp
   #[===============================[
      main sources:
        subdir: perflog
@@ -749,6 +757,7 @@ if (tests)
     src/test/app/Taker_test.cpp
     src/test/app/TheoreticalQuality_test.cpp
     src/test/app/Ticket_test.cpp
+    src/test/app/Touch_test.cpp
     src/test/app/Transaction_ordering_test.cpp
     src/test/app/TrustAndBalance_test.cpp
     src/test/app/TxQ_test.cpp
@@ -988,6 +997,7 @@ if (tests)
     src/test/rpc/AccountTx_test.cpp
     src/test/rpc/AmendmentBlocked_test.cpp
     src/test/rpc/Book_test.cpp
+    src/test/rpc/Catalogue_test.cpp
     src/test/rpc/DepositAuthorized_test.cpp
     src/test/rpc/DeliveredAmount_test.cpp
     src/test/rpc/Feature_test.cpp

Builds/CMake/deps/Boost.cmake

@@ -1,14 +1,16 @@
 #[===================================================================[
    NIH dep: boost
 #]===================================================================]
-
 if((NOT DEFINED BOOST_ROOT) AND(DEFINED ENV{BOOST_ROOT}))
   set(BOOST_ROOT $ENV{BOOST_ROOT})
 endif()
+if((NOT DEFINED BOOST_LIBRARYDIR) AND(DEFINED ENV{BOOST_LIBRARYDIR}))
+  set(BOOST_LIBRARYDIR $ENV{BOOST_LIBRARYDIR})
+endif()
 file(TO_CMAKE_PATH "${BOOST_ROOT}" BOOST_ROOT)
 if(WIN32 OR CYGWIN)
   # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders
-  if(DEFINED BOOST_ROOT)
+  if((NOT DEFINED BOOST_LIBRARYDIR) AND (DEFINED BOOST_ROOT))
     if(IS_DIRECTORY ${BOOST_ROOT}/stage64/lib)
       set(BOOST_LIBRARYDIR ${BOOST_ROOT}/stage64/lib)
     elseif(IS_DIRECTORY ${BOOST_ROOT}/stage/lib)
@@ -55,6 +57,7 @@ find_package(Boost 1.86 REQUIRED
     program_options
     regex
     system
+    iostreams
     thread)
 
 add_library(ripple_boost INTERFACE)
@@ -74,6 +77,7 @@ target_link_libraries(ripple_boost
     Boost::coroutine
     Boost::date_time
     Boost::filesystem
+    Boost::iostreams
     Boost::program_options
     Boost::regex
     Boost::system

Builds/CMake/deps/FindBoost.cmake

@@ -248,6 +248,7 @@ include(FindPackageHandleStandardArgs)
 # Save project's policies
 cmake_policy(PUSH)
 cmake_policy(SET CMP0057 NEW) # if IN_LIST
+#cmake_policy(SET CMP0144 NEW)
 
 #-------------------------------------------------------------------------------
 # Before we go searching, check whether a boost cmake package is available, unless
@@ -969,7 +970,24 @@ function(_Boost_COMPONENT_DEPENDENCIES component _ret)
       set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono date_time atomic)
       set(_Boost_WSERIALIZATION_DEPENDENCIES serialization)
     endif()
-    if(NOT Boost_VERSION_STRING VERSION_LESS 1.77.0)
+    
+    # Special handling for Boost 1.86.0 and higher
+    if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0)
+      # Explicitly set these for Boost 1.86
+      set(_Boost_IOSTREAMS_DEPENDENCIES "")  # No dependencies for iostreams in 1.86
+      
+      # Debug output to help diagnose the issue
+      if(Boost_DEBUG)
+        message(STATUS "Using special dependency settings for Boost 1.86.0+")
+        message(STATUS "Component: ${component}, uppercomponent: ${uppercomponent}")
+        message(STATUS "Boost_VERSION_STRING: ${Boost_VERSION_STRING}")
+        message(STATUS "BOOST_ROOT: $ENV{BOOST_ROOT}")
+        message(STATUS "BOOST_LIBRARYDIR: $ENV{BOOST_LIBRARYDIR}")
+      endif()
+    endif()
+    
+    # Only show warning for versions beyond what we've defined
+    if(NOT Boost_VERSION_STRING VERSION_LESS 1.87.0)
       message(WARNING "New Boost version may have incorrect or missing dependencies and imported targets")
     endif()
   endif()
@@ -1879,6 +1897,18 @@ foreach(COMPONENT ${Boost_FIND_COMPONENTS})
     list(INSERT _boost_LIBRARY_SEARCH_DIRS_RELEASE 0 ${Boost_LIBRARY_DIR_DEBUG})
   endif()
 
+  if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0)
+    if(BOOST_LIBRARYDIR AND EXISTS "${BOOST_LIBRARYDIR}")
+      # Clear existing search paths and use only BOOST_LIBRARYDIR 
+      set(_boost_LIBRARY_SEARCH_DIRS_RELEASE "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH)
+      set(_boost_LIBRARY_SEARCH_DIRS_DEBUG "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH)
+        
+      if(Boost_DEBUG)
+        message(STATUS "Boost 1.86: Setting library search dirs to BOOST_LIBRARYDIR: ${BOOST_LIBRARYDIR}")
+      endif()
+    endif()
+  endif()
+
   # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing.
   string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS_RELEASE}")
 

Builds/CMake/deps/gRPC.cmake

@@ -74,7 +74,11 @@ else ()
       if (NOT _location)
         message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")
       endif ()
-      add_library (c-ares::cares ${_static} IMPORTED GLOBAL)
+      if(${_location} MATCHES "\\.a$")
+        add_library(c-ares::cares STATIC IMPORTED GLOBAL)
+      else()
+        add_library(c-ares::cares SHARED IMPORTED GLOBAL)
+      endif()      
       set_target_properties (c-ares::cares PROPERTIES
         IMPORTED_LOCATION ${_location}
         INTERFACE_INCLUDE_DIRECTORIES "${${_prefix}_INCLUDE_DIRS}"
@@ -204,6 +208,7 @@ else ()
       CMAKE_ARGS
         -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
         -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_STANDARD=17
         $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
         $<$<BOOL:${CMAKE_TOOLCHAIN_FILE}>:-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}>
         $<$<BOOL:${VCPKG_TARGET_TRIPLET}>:-DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}>

CMakeLists.txt

@@ -1,14 +1,18 @@
 cmake_minimum_required (VERSION 3.16)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 
 if (POLICY CMP0074)
   cmake_policy(SET CMP0074 NEW)
 endif ()
 
-project (rippled)
-set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+if(POLICY CMP0144)
+  cmake_policy(SET CMP0144 NEW)
+endif()
 
+project (rippled)
 set(Boost_NO_BOOST_CMAKE ON)
 
 # make GIT_COMMIT_HASH define available to all sources

README.md

@@ -67,5 +67,5 @@ git-subtree. See those directories' README files for more details.
   - [explorer.xahau.network](https://explorer.xahau.network)
 - **Testnet & Faucet**: Test applications and obtain test XAH at [xahau-test.net](https://xahau-test.net) and use the testnet explorer at [explorer.xahau.network](https://explorer.xahau.network).
 - **Supporting Wallets**: A list of wallets that support XAH and Xahau-based assets.
-  - [Xumm](https://xumm.app)
+  - [Xaman](https://xaman.app)
   - [Crossmark](https://crossmark.io)

build-core.sh

@@ -1,4 +1,11 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
+
+set -e
 
 echo "START INSIDE CONTAINER - CORE"
 
@@ -23,7 +30,7 @@ fi
 perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake &&
 mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old &&
 echo "find_package(LLVM REQUIRED CONFIG)
-message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\")
+message(STATUS \"Found LLVM \${LLVM_PACKAGE_VERSION}\")
 message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\")
 add_library (wasmedge STATIC IMPORTED GLOBAL)
 set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB})

build-full.sh

@@ -1,4 +1,11 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
+
+set -e
 
 echo "START INSIDE CONTAINER - FULL"
 
@@ -19,7 +26,7 @@ yum-config-manager --disable centos-sclo-sclo
 ####
 
 cd /io;
-mkdir src/certs;
+mkdir -p src/certs;
 curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h;
 if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ]
 then
@@ -66,8 +73,8 @@ then
     #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp &&
     sed -i "s/#include <ripple\/net\/RegisterSSLCerts.h>/\0\n#include <certs\/certbundle.h>/g" src/ripple/net/impl/RegisterSSLCerts.cpp
 fi
-mkdir .nih_c;
-mkdir .nih_toolchain;
+mkdir -p .nih_c;
+mkdir -p .nih_toolchain;
 cd .nih_toolchain &&
 yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static -y \
   devtoolset-7-gcc-c++ \
@@ -92,7 +99,7 @@ pwd &&
 tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ &&
 echo "-- Install Boost 1.86.0 --" &&
 pwd &&
-( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) &&
+( wget -nc -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) &&
 tar -xzf boost_1_86_0.tar.gz &&
 cd boost_1_86_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
 cd ../ &&
@@ -115,7 +122,7 @@ tar -xf libunwind-13.0.1.src.tar.xz &&
 cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ &&
 cd lld-13.0.1.src &&
 rm -rf build CMakeCache.txt &&
-mkdir build &&
+mkdir -p build &&
 cd build &&
 cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release &&
 make -j$3 install &&
@@ -125,7 +132,7 @@ cd ../../ &&
 echo "-- Build WasmEdge --" &&
 ( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) &&
 cd WasmEdge-0.11.2 &&
-( mkdir build; echo "" ) &&
+( mkdir -p build; echo "" ) &&
 cd build &&
 export BOOST_ROOT="/usr/local/src/boost_1_86_0" &&
 export Boost_LIBRARY_DIRS="/usr/local/lib" &&

cfg/rippled-example.cfg

@@ -1056,7 +1056,18 @@
 #       Cassandra is an alternative backend to be used only with Reporting Mode.
 #       See the Reporting Mode section for more details about Reporting Mode.
 #
-#   Required keys for NuDB and RocksDB:
+#   type = RWDB
+#
+#       RWDB is a high-performance memory store written by XRPL-Labs and optimized
+#       for xahaud. RWDB is NOT persistent and the data will be lost on restart.
+#       RWDB is recommended for Validator and Peer nodes that are not required to 
+#       store history.
+#
+#       RWDB maintains its high speed regardless of the amount of history
+#       stored. Online delete should NOT be used instead RWDB will use the 
+#       ledger_history config value to determine how many ledgers to keep in memory.
+#
+#   Required keys for NuDB, RWDB and RocksDB:
 #
 #       path                Location to store the database
 #
@@ -1112,7 +1123,8 @@
 #       online_delete       Minimum value of 256. Enable automatic purging
 #                           of older ledger information. Maintain at least this
 #                           number of ledger records online. Must be greater
-#                           than or equal to ledger_history.
+#                           than or equal to ledger_history. If using RWDB 
+#                           this value is ignored.
 #
 #       These keys modify the behavior of online_delete, and thus are only
 #       relevant if online_delete is defined and non-zero:

cfg/rippled-standalone.cfg

@@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite
 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1
 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1
-36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1
+215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2
+0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit
+7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M
+4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete
+EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
+8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
+88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911

docker-unit-tests.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
+echo "Mounting $(pwd)/io in ubuntu and running unit tests"
 docker run --rm -i -v $(pwd):/io ubuntu sh -c '/io/release-build/xahaud -u'
-

hook/generate_sfcodes.sh

@@ -0,0 +1,29 @@
+#/bin/bash
+RIPPLED_ROOT="../src/ripple"
+echo '// For documentation please see: https://xrpl-hooks.readme.io/reference/'
+echo '// Generated using generate_sfcodes.sh'
+cat $RIPPLED_ROOT/protocol/impl/SField.cpp | grep -E '^CONSTRUCT_' |
+    sed 's/UINT16/1/g' |
+    sed 's/UINT32/2/g' |
+    sed 's/UINT64/3/g' |
+    sed 's/HASH128/4/g' |
+    sed 's/HASH256/5/g' |
+    sed 's/UINT128/4/g' |
+    sed 's/UINT256/5/g' |
+    sed 's/AMOUNT/6/g' |
+    sed 's/VL/7/g' | sed 's/Import7/ImportVL/g' |
+    sed 's/ACCOUNT/8/g' |
+    sed 's/OBJECT/14/g' |
+    sed 's/ARRAY/15/g' |
+    sed 's/UINT8/16/g' |
+    sed 's/HASH160/17/g' |
+    sed 's/UINT160/17/g' |
+    sed 's/PATHSET/18/g' |
+    sed 's/VECTOR256/19/g' |
+    sed 's/UINT96/20/g' |
+    sed 's/UINT192/21/g' |
+    sed 's/UINT384/22/g' |
+    sed 's/UINT512/23/g' |
+    grep -Eo '"([^"]+)", *([0-9]+), *([0-9]+)' |
+    sed 's/"//g' | sed 's/ *//g' | sed 's/,/ /g' |
+    awk '{print ("#define sf"$1" (("$2"U << 16U) + "$3"U)")}'

hook/macro.h

@@ -1,5 +1,5 @@
 /**
- * These are helper macros for writing hooks, all of them are optional as is including hookmacro.h at all
+ * These are helper macros for writing hooks, all of them are optional as is including macro.h at all
  */
 
 #include <stdint.h>

hook/sfcodes.h

@@ -60,7 +60,10 @@
 #define sfBurnedNFTokens ((2U << 16U) + 44U)
 #define sfHookStateCount ((2U << 16U) + 45U)
 #define sfEmitGeneration ((2U << 16U) + 46U)
-#define sfLockCount ((2U << 16U) + 47U)
+#define sfLockCount ((2U << 16U) + 49U)
+#define sfFirstNFTokenSequence ((2U << 16U) + 50U)
+#define sfXahauActivationLgrSeq ((2U << 16U) + 96U)
+#define sfImportSequence ((2U << 16U) + 97U)
 #define sfRewardTime ((2U << 16U) + 98U)
 #define sfRewardLgrFirst ((2U << 16U) + 99U)
 #define sfRewardLgrLast ((2U << 16U) + 100U)
@@ -80,6 +83,8 @@
 #define sfHookInstructionCount ((3U << 16U) + 17U)
 #define sfHookReturnCode ((3U << 16U) + 18U)
 #define sfReferenceCount ((3U << 16U) + 19U)
+#define sfAccountIndex ((3U << 16U) + 98U)
+#define sfAccountCount ((3U << 16U) + 99U)
 #define sfRewardAccumulator ((3U << 16U) + 100U)
 #define sfEmailHash ((4U << 16U) + 1U)
 #define sfTakerPaysCurrency ((10U << 16U) + 1U)
@@ -120,6 +125,9 @@
 #define sfOfferID ((5U << 16U) + 34U)
 #define sfEscrowID ((5U << 16U) + 35U)
 #define sfURITokenID ((5U << 16U) + 36U)
+#define sfGovernanceFlags ((5U << 16U) + 99U)
+#define sfGovernanceMarks ((5U << 16U) + 98U)
+#define sfEmittedTxnID ((5U << 16U) + 97U)
 #define sfAmount ((6U << 16U) + 1U)
 #define sfBalance ((6U << 16U) + 2U)
 #define sfLimitAmount ((6U << 16U) + 3U)
@@ -136,6 +144,9 @@
 #define sfNFTokenBrokerFee ((6U << 16U) + 19U)
 #define sfHookCallbackFee ((6U << 16U) + 20U)
 #define sfLockedBalance ((6U << 16U) + 21U)
+#define sfBaseFeeDrops ((6U << 16U) + 22U)
+#define sfReserveBaseDrops ((6U << 16U) + 23U)
+#define sfReserveIncrementDrops ((6U << 16U) + 24U)
 #define sfPublicKey ((7U << 16U) + 1U)
 #define sfMessageKey ((7U << 16U) + 2U)
 #define sfSigningPubKey ((7U << 16U) + 3U)
@@ -171,11 +182,13 @@
 #define sfNFTokenMinter ((8U << 16U) + 9U)
 #define sfEmitCallback ((8U << 16U) + 10U)
 #define sfHookAccount ((8U << 16U) + 16U)
+#define sfInform ((8U << 16U) + 99U)
 #define sfIndexes ((19U << 16U) + 1U)
 #define sfHashes ((19U << 16U) + 2U)
 #define sfAmendments ((19U << 16U) + 3U)
 #define sfNFTokenOffers ((19U << 16U) + 4U)
 #define sfHookNamespaces ((19U << 16U) + 5U)
+#define sfURITokenIDs ((19U << 16U) + 99U)
 #define sfPaths ((18U << 16U) + 1U)
 #define sfTransactionMetaData ((14U << 16U) + 2U)
 #define sfCreatedNode ((14U << 16U) + 3U)
@@ -198,6 +211,12 @@
 #define sfHookDefinition ((14U << 16U) + 22U)
 #define sfHookParameter ((14U << 16U) + 23U)
 #define sfHookGrant ((14U << 16U) + 24U)
+#define sfGenesisMint ((14U << 16U) + 96U)
+#define sfActiveValidator ((14U << 16U) + 95U)
+#define sfImportVLKey ((14U << 16U) + 94U)
+#define sfHookEmission ((14U << 16U) + 93U)
+#define sfMintURIToken ((14U << 16U) + 92U)
+#define sfAmountEntry ((14U << 16U) + 91U)
 #define sfSigners ((15U << 16U) + 3U)
 #define sfSignerEntries ((15U << 16U) + 4U)
 #define sfTemplate ((15U << 16U) + 5U)
@@ -212,4 +231,8 @@
 #define sfHookExecutions ((15U << 16U) + 18U)
 #define sfHookParameters ((15U << 16U) + 19U)
 #define sfHookGrants ((15U << 16U) + 20U)
+#define sfGenesisMints ((15U << 16U) + 96U)
 #define sfActiveValidators ((15U << 16U) + 95U)
+#define sfImportVLKeys ((15U << 16U) + 94U)
+#define sfHookEmissions ((15U << 16U) + 93U)
+#define sfAmounts ((15U << 16U) + 92U)

release-builder.sh

@@ -1,4 +1,9 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
 
 echo "START BUILDING (HOST)"
 
@@ -12,7 +17,26 @@ if [[ "$GITHUB_REPOSITORY" == "" ]]; then
   BUILD_CORES=8
 fi
 
-CONTAINER_NAME=xahaud_cached_builder_$(echo "$GITHUB_ACTOR" | awk '{print tolower($0)}')
+EXIT_IF_CONTAINER_RUNNING=${EXIT_IF_CONTAINER_RUNNING:-1}
+# Ensure still works outside of GH Actions by setting these to /dev/null
+# GA will run this script and then delete it at the end of the job
+JOB_CLEANUP_SCRIPT=${JOB_CLEANUP_SCRIPT:-/dev/null}
+NORMALIZED_WORKFLOW=$(echo "$GITHUB_WORKFLOW" | tr -c 'a-zA-Z0-9' '-')
+NORMALIZED_REF=$(echo "$GITHUB_REF" | tr -c 'a-zA-Z0-9' '-')
+CONTAINER_NAME="xahaud_cached_builder_${NORMALIZED_WORKFLOW}-${NORMALIZED_REF}"
+
+# Check if the container is already running
+if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+    echo "⚠️ A running container (${CONTAINER_NAME}) was detected."
+
+    if [[ "$EXIT_IF_CONTAINER_RUNNING" -eq 1 ]]; then
+        echo "❌ EXIT_IF_CONTAINER_RUNNING is set. Exiting."
+        exit 1
+    else
+        echo "🛑 Stopping the running container: ${CONTAINER_NAME}"
+        docker stop "${CONTAINER_NAME}"
+    fi
+fi
 
 echo "-- BUILD CORES:       $BUILD_CORES"
 echo "-- GITHUB_REPOSITORY: $GITHUB_REPOSITORY"
@@ -36,7 +60,8 @@ fi
 
 STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME |wc -l)
 
-if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then
+#if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then
+if false; then
   echo "Static container, execute in static container to have max. cache"
   docker start $CONTAINER_NAME
   docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
@@ -54,6 +79,8 @@ else
     # GH Action, runner
     echo "GH Action, runner, clean & re-create create persistent container"
     docker rm -f $CONTAINER_NAME
+    echo "echo 'Stopping container: $CONTAINER_NAME'" >> "$JOB_CLEANUP_SCRIPT"
+    echo "docker stop --time=15 \"$CONTAINER_NAME\" || echo 'Failed to stop container or container not running'" >> "$JOB_CLEANUP_SCRIPT"
     docker run -di --user 0:$(id -g) --name $CONTAINER_NAME -v /data/builds:/data/builds -v `pwd`:/io --network host ghcr.io/foobarwidget/holy-build-box-x64 /hbb_exe/activate-exec bash
     docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-full.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"
     docker stop $CONTAINER_NAME

src/ripple/app/consensus/RCLConsensus.cpp

@@ -186,7 +186,7 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx)
     if (app_.getHashRouter().shouldRelay(tx.id()))
     {
         JLOG(j_.debug()) << "Relaying disputed tx " << tx.id();
-        auto const slice = tx.tx_.slice();
+        auto const slice = tx.tx_->slice();
         protocol::TMTransaction msg;
         msg.set_rawtransaction(slice.data(), slice.size());
         msg.set_status(protocol::tsNEW);
@@ -330,7 +330,7 @@ RCLConsensus::Adaptor::onClose(
         tx.first->add(s);
         initialSet->addItem(
             SHAMapNodeType::tnTRANSACTION_NM,
-            SHAMapItem(tx.first->getTransactionID(), s.slice()));
+            make_shamapitem(tx.first->getTransactionID(), s.slice()));
     }
 
     // Add pseudo-transactions to the set
@@ -374,7 +374,8 @@ RCLConsensus::Adaptor::onClose(
         RCLCensorshipDetector<TxID, LedgerIndex>::TxIDSeqVec proposed;
 
         initialSet->visitLeaves(
-            [&proposed, seq](std::shared_ptr<SHAMapItem const> const& item) {
+            [&proposed,
+             seq](boost::intrusive_ptr<SHAMapItem const> const& item) {
                 proposed.emplace_back(item->key(), seq);
             });
 
@@ -539,7 +540,7 @@ RCLConsensus::Adaptor::doAccept(
         std::vector<TxID> accepted;
 
         result.txns.map_->visitLeaves(
-            [&accepted](std::shared_ptr<SHAMapItem const> const& item) {
+            [&accepted](boost::intrusive_ptr<SHAMapItem const> const& item) {
                 accepted.push_back(item->key());
             });
 
@@ -614,7 +615,7 @@ RCLConsensus::Adaptor::doAccept(
                         << "Test applying disputed transaction that did"
                         << " not get in " << dispute.tx().id();
 
-                    SerialIter sit(dispute.tx().tx_.slice());
+                    SerialIter sit(dispute.tx().tx_->slice());
                     auto txn = std::make_shared<STTx const>(sit);
 
                     // Disputed pseudo-transactions that were not accepted

src/ripple/app/consensus/RCLCxTx.h

@@ -42,7 +42,7 @@ public:
 
         @param txn The transaction to wrap
     */
-    RCLCxTx(SHAMapItem const& txn) : tx_{txn}
+    RCLCxTx(boost::intrusive_ptr<SHAMapItem const> txn) : tx_(std::move(txn))
     {
     }
 
@@ -50,11 +50,11 @@ public:
     ID const&
     id() const
     {
-        return tx_.key();
+        return tx_->key();
     }
 
     //! The SHAMapItem that represents the transaction.
-    SHAMapItem const tx_;
+    boost::intrusive_ptr<SHAMapItem const> tx_;
 };
 
 /** Represents a set of transactions in RCLConsensus.
@@ -90,8 +90,7 @@ public:
         bool
         insert(Tx const& t)
         {
-            return map_->addItem(
-                SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{t.tx_});
+            return map_->addItem(SHAMapNodeType::tnTRANSACTION_NM, t.tx_);
         }
 
         /** Remove a transaction from the set.
@@ -145,7 +144,7 @@ public:
               code uses the shared_ptr semantics to know whether the find
               was successful and properly creates a Tx as needed.
     */
-    std::shared_ptr<const SHAMapItem> const&
+    boost::intrusive_ptr<SHAMapItem const> const&
     find(Tx::ID const& entry) const
     {
         return map_->peekItem(entry);

src/ripple/app/hook/Guard.h

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <map>
 #include <memory>
+#include <optional>
 #include <ostream>
 #include <stack>
 #include <string>
@@ -271,7 +272,19 @@ check_guard(
     int guard_func_idx,
     int last_import_idx,
     GuardLog guardLog,
-    std::string guardLogAccStr)
+    std::string guardLogAccStr,
+    /* RH NOTE:
+     * rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
+     * and update 3 is 0x04 ideally at rule version 3 all bits so far are set
+     * (0b111) so the ruleVersion = 7, however if a specific rule update must be
+     * rolled back due to unforeseen behaviour then this may no longer be the
+     * case. using a bit field here leaves us flexible to rollback changes that
+     * might have unforeseen consequences, without also rolling back further
+     * changes that are fine.
+     */
+    uint64_t rulesVersion = 0
+
+)
 {
 #define MAX_GUARD_CALLS 1024
     uint32_t guard_count = 0;
@@ -621,11 +634,17 @@ check_guard(
             }
             else if (fc_type == 10)  // memory.copy
             {
+                if (rulesVersion & 0x02U)
+                    GUARD_ERROR("Memory.copy instruction is not allowed.");
+
                 REQUIRE(2);
                 ADVANCE(2);
             }
             else if (fc_type == 11)  // memory.fill
             {
+                if (rulesVersion & 0x02U)
+                    GUARD_ERROR("Memory.fill instruction is not allowed.");
+
                 ADVANCE(1);
             }
             else if (fc_type <= 7)  // numeric instructions
@@ -807,6 +826,15 @@ validateGuards(
     std::vector<uint8_t> const& wasm,
     GuardLog guardLog,
     std::string guardLogAccStr,
+    /* RH NOTE:
+     * rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
+     * and update 3 is 0x04 ideally at rule version 3 all bits so far are set
+     * (0b111) so the ruleVersion = 7, however if a specific rule update must be
+     * rolled back due to unforeseen behaviour then this may no longer be the
+     * case. using a bit field here leaves us flexible to rollback changes that
+     * might have unforeseen consequences, without also rolling back further
+     * changes that are fine.
+     */
     uint64_t rulesVersion = 0)
 {
     uint64_t byteCount = wasm.size();
@@ -1477,7 +1505,8 @@ validateGuards(
                     guard_import_number,
                     last_import_number,
                     guardLog,
-                    guardLogAccStr);
+                    guardLogAccStr,
+                    rulesVersion);
 
                 if (!valid)
                     return {};

src/ripple/app/hook/guard_checker.cpp

@@ -79,7 +79,7 @@ main(int argc, char** argv)
 
     close(fd);
 
-    auto result = validateGuards(hook, std::cout, "", 1);
+    auto result = validateGuards(hook, std::cout, "", 3);
 
     if (!result)
     {

src/ripple/app/hook/impl/applyHook.cpp

@@ -1217,9 +1217,10 @@ hook::apply(
              .hookParamOverrides = hookParamOverrides,
              .hookParams = hookParams,
              .hookSkips = {},
-             .exitType =
-                 hook_api::ExitType::ROLLBACK,  // default is to rollback unless
-                                                // hook calls accept()
+             .exitType = applyCtx.view().rules().enabled(fixXahauV3)
+                 ? hook_api::ExitType::UNSET
+                 : hook_api::ExitType::ROLLBACK,  // default is to rollback
+                                                  // unless hook calls accept()
              .exitReason = std::string(""),
              .exitCode = -1,
              .hasCallback = hasCallback,
@@ -4617,6 +4618,8 @@ DEFINE_HOOK_FUNCTION(
     }
     catch (std::exception& e)
     {
+        JLOG(j.trace()) << "HookInfo[" << HC_ACC()
+                        << "]: etxn_fee_base exception: " << e.what();
         return INVALID_TXN;
     }
 
@@ -4788,7 +4791,7 @@ DEFINE_HOOK_FUNCTION(
 
     if (float1 == 0)
     {
-        j.trace() << "HookTrace[" << HC_ACC() << "]:"
+        j.trace() << "HookTrace[" << HC_ACC() << "]: "
                   << (read_len == 0
                           ? ""
                           : std::string_view(
@@ -5402,7 +5405,7 @@ DEFINE_HOOK_FUNCTION(
 const int64_t float_one_internal = make_float(1000000000000000ull, -15, false);
 
 inline int64_t
-float_divide_internal(int64_t float1, int64_t float2)
+float_divide_internal(int64_t float1, int64_t float2, bool hasFix)
 {
     RETURN_IF_INVALID_FLOAT(float1);
     RETURN_IF_INVALID_FLOAT(float2);
@@ -5455,8 +5458,16 @@ float_divide_internal(int64_t float1, int64_t float2)
     while (man2 > 0)
     {
         int i = 0;
-        for (; man1 > man2; man1 -= man2, ++i)
-            ;
+        if (hasFix)
+        {
+            for (; man1 >= man2; man1 -= man2, ++i)
+                ;
+        }
+        else
+        {
+            for (; man1 > man2; man1 -= man2, ++i)
+                ;
+        }
 
         man3 *= 10;
         man3 += i;
@@ -5476,7 +5487,8 @@ DEFINE_HOOK_FUNCTION(int64_t, float_divide, int64_t float1, int64_t float2)
     HOOK_SETUP();  // populates memory_ctx, memory, memory_length, applyCtx,
                    // hookCtx on current stack
 
-    return float_divide_internal(float1, float2);
+    bool const hasFix = view.rules().enabled(fixFloatDivide);
+    return float_divide_internal(float1, float2, hasFix);
 
     HOOK_TEARDOWN();
 }
@@ -5495,7 +5507,9 @@ DEFINE_HOOK_FUNCTION(int64_t, float_invert, int64_t float1)
         return DIVISION_BY_ZERO;
     if (float1 == float_one_internal)
         return float_one_internal;
-    return float_divide_internal(float_one_internal, float1);
+
+    bool const fixV3 = view.rules().enabled(fixFloatDivide);
+    return float_divide_internal(float_one_internal, float1, fixV3);
 
     HOOK_TEARDOWN();
 }

src/ripple/app/ledger/Ledger.cpp

@@ -119,9 +119,8 @@ public:
     sles_type::value_type
     dereference() const override
     {
-        auto const item = *iter_;
-        SerialIter sit(item.slice());
-        return std::make_shared<SLE const>(sit, item.key());
+        SerialIter sit(iter_->slice());
+        return std::make_shared<SLE const>(sit, iter_->key());
     }
 };
 
@@ -168,7 +167,7 @@ public:
     txs_type::value_type
     dereference() const override
     {
-        auto const item = *iter_;
+        auto const& item = *iter_;
         if (metadata_)
             return deserializeTxPlusMeta(item);
         return {deserializeTx(item), nullptr};
@@ -183,8 +182,8 @@ Ledger::Ledger(
     std::vector<uint256> const& amendments,
     Family& family)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(SHAMapType::TRANSACTION, family))
-    , stateMap_(std::make_shared<SHAMap>(SHAMapType::STATE, family))
+    , txMap_(SHAMapType::TRANSACTION, family)
+    , stateMap_(SHAMapType::STATE, family)
     , rules_{config.features}
     , j_(beast::Journal(beast::Journal::getNullSink()))
 {
@@ -247,7 +246,7 @@ Ledger::Ledger(
         rawInsert(sle);
     }
 
-    stateMap_->flushDirty(hotACCOUNT_NODE);
+    stateMap_.flushDirty(hotACCOUNT_NODE);
     setImmutable();
 }
 
@@ -259,12 +258,8 @@ Ledger::Ledger(
     Family& family,
     beast::Journal j)
     : mImmutable(true)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          info.txHash,
-          family))
-    , stateMap_(
-          std::make_shared<SHAMap>(SHAMapType::STATE, info.accountHash, family))
+    , txMap_(SHAMapType::TRANSACTION, info.txHash, family)
+    , stateMap_(SHAMapType::STATE, info.accountHash, family)
     , rules_(config.features)
     , info_(info)
     , j_(j)
@@ -272,7 +267,7 @@ Ledger::Ledger(
     loaded = true;
 
     if (info_.txHash.isNonZero() &&
-        !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
+        !txMap_.fetchRoot(SHAMapHash{info_.txHash}, nullptr))
     {
         if (config.reporting())
         {
@@ -284,7 +279,7 @@ Ledger::Ledger(
     }
 
     if (info_.accountHash.isNonZero() &&
-        !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
+        !stateMap_.fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
     {
         if (config.reporting())
         {
@@ -295,8 +290,8 @@ Ledger::Ledger(
         JLOG(j.warn()) << "Don't have state data root for ledger" << info_.seq;
     }
 
-    txMap_->setImmutable();
-    stateMap_->setImmutable();
+    txMap_.setImmutable();
+    stateMap_.setImmutable();
 
     defaultFees(config);
     if (!setup())
@@ -310,13 +305,25 @@ Ledger::Ledger(
     }
 }
 
+Ledger::Ledger(
+    LedgerInfo& info,
+    Config const& config,
+    Family& family,
+    SHAMap const& baseState)
+    : mImmutable(false)
+    , info_(info)
+    , txMap_(SHAMapType::TRANSACTION, family)
+    , stateMap_(baseState, true)
+    , rules_{config.features}
+    , j_(beast::Journal(beast::Journal::getNullSink()))
+{
+}
+
 // Create a new ledger that follows this one
 Ledger::Ledger(Ledger const& prevLedger, NetClock::time_point closeTime)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          prevLedger.stateMap_->family()))
-    , stateMap_(prevLedger.stateMap_->snapShot(true))
+    , txMap_(SHAMapType::TRANSACTION, prevLedger.txMap_.family())
+    , stateMap_(prevLedger.stateMap_, true)
     , fees_(prevLedger.fees_)
     , rules_(prevLedger.rules_)
     , j_(beast::Journal(beast::Journal::getNullSink()))
@@ -345,12 +352,8 @@ Ledger::Ledger(Ledger const& prevLedger, NetClock::time_point closeTime)
 
 Ledger::Ledger(LedgerInfo const& info, Config const& config, Family& family)
     : mImmutable(true)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          info.txHash,
-          family))
-    , stateMap_(
-          std::make_shared<SHAMap>(SHAMapType::STATE, info.accountHash, family))
+    , txMap_(SHAMapType::TRANSACTION, info.txHash, family)
+    , stateMap_(SHAMapType::STATE, info.accountHash, family)
     , rules_{config.features}
     , info_(info)
     , j_(beast::Journal(beast::Journal::getNullSink()))
@@ -364,8 +367,8 @@ Ledger::Ledger(
     Config const& config,
     Family& family)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(SHAMapType::TRANSACTION, family))
-    , stateMap_(std::make_shared<SHAMap>(SHAMapType::STATE, family))
+    , txMap_(SHAMapType::TRANSACTION, family)
+    , stateMap_(SHAMapType::STATE, family)
     , rules_{config.features}
     , j_(beast::Journal(beast::Journal::getNullSink()))
 {
@@ -383,19 +386,32 @@ Ledger::setImmutable(bool rehash)
     // place the hash transitions to valid
     if (!mImmutable && rehash)
     {
-        info_.txHash = txMap_->getHash().as_uint256();
-        info_.accountHash = stateMap_->getHash().as_uint256();
+        info_.txHash = txMap_.getHash().as_uint256();
+        info_.accountHash = stateMap_.getHash().as_uint256();
     }
 
     if (rehash)
         info_.hash = calculateLedgerHash(info_);
 
     mImmutable = true;
-    txMap_->setImmutable();
-    stateMap_->setImmutable();
+    txMap_.setImmutable();
+    stateMap_.setImmutable();
     setup();
 }
 
+// raw setters for catalogue
+void
+Ledger::setCloseFlags(int closeFlags)
+{
+    info_.closeFlags = closeFlags;
+}
+
+void
+Ledger::setDrops(uint64_t drops)
+{
+    info_.drops = drops;
+}
+
 void
 Ledger::setAccepted(
     NetClock::time_point closeTime,
@@ -415,8 +431,8 @@ bool
 Ledger::addSLE(SLE const& sle)
 {
     auto const s = sle.getSerializer();
-    SHAMapItem item(sle.key(), s.slice());
-    return stateMap_->addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(item));
+    return stateMap_.addItem(
+        SHAMapNodeType::tnACCOUNT_STATE, make_shamapitem(sle.key(), s.slice()));
 }
 
 //------------------------------------------------------------------------------
@@ -451,20 +467,20 @@ bool
 Ledger::exists(Keylet const& k) const
 {
     // VFALCO NOTE Perhaps check the type for debug builds?
-    return stateMap_->hasItem(k.key);
+    return stateMap_.hasItem(k.key);
 }
 
 bool
 Ledger::exists(uint256 const& key) const
 {
-    return stateMap_->hasItem(key);
+    return stateMap_.hasItem(key);
 }
 
 std::optional<uint256>
 Ledger::succ(uint256 const& key, std::optional<uint256> const& last) const
 {
-    auto item = stateMap_->upper_bound(key);
-    if (item == stateMap_->end())
+    auto item = stateMap_.upper_bound(key);
+    if (item == stateMap_.end())
         return std::nullopt;
     if (last && item->key() >= last)
         return std::nullopt;
@@ -479,7 +495,7 @@ Ledger::read(Keylet const& k) const
         assert(false);
         return nullptr;
     }
-    auto const& item = stateMap_->peekItem(k.key);
+    auto const& item = stateMap_.peekItem(k.key);
     if (!item)
         return nullptr;
     auto sle = std::make_shared<SLE>(SerialIter{item->slice()}, item->key());
@@ -493,45 +509,44 @@ Ledger::read(Keylet const& k) const
 auto
 Ledger::slesBegin() const -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->begin());
+    return std::make_unique<sles_iter_impl>(stateMap_.begin());
 }
 
 auto
 Ledger::slesEnd() const -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->end());
+    return std::make_unique<sles_iter_impl>(stateMap_.end());
 }
 
 auto
 Ledger::slesUpperBound(uint256 const& key) const
     -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->upper_bound(key));
+    return std::make_unique<sles_iter_impl>(stateMap_.upper_bound(key));
 }
 
 auto
 Ledger::txsBegin() const -> std::unique_ptr<txs_type::iter_base>
 {
-    return std::make_unique<txs_iter_impl>(!open(), txMap_->begin());
+    return std::make_unique<txs_iter_impl>(!open(), txMap_.begin());
 }
 
 auto
 Ledger::txsEnd() const -> std::unique_ptr<txs_type::iter_base>
 {
-    return std::make_unique<txs_iter_impl>(!open(), txMap_->end());
+    return std::make_unique<txs_iter_impl>(!open(), txMap_.end());
 }
 
 bool
 Ledger::txExists(uint256 const& key) const
 {
-    return txMap_->hasItem(key);
+    return txMap_.hasItem(key);
 }
 
 auto
 Ledger::txRead(key_type const& key) const -> tx_type
 {
-    assert(txMap_);
-    auto const& item = txMap_->peekItem(key);
+    auto const& item = txMap_.peekItem(key);
     if (!item)
         return {};
     if (!open())
@@ -548,7 +563,7 @@ Ledger::digest(key_type const& key) const -> std::optional<digest_type>
     SHAMapHash digest;
     // VFALCO Unfortunately this loads the item
     //        from the NodeStore needlessly.
-    if (!stateMap_->peekItem(key, digest))
+    if (!stateMap_.peekItem(key, digest))
         return std::nullopt;
     return digest.as_uint256();
 }
@@ -558,14 +573,14 @@ Ledger::digest(key_type const& key) const -> std::optional<digest_type>
 void
 Ledger::rawErase(std::shared_ptr<SLE> const& sle)
 {
-    if (!stateMap_->delItem(sle->key()))
+    if (!stateMap_.delItem(sle->key()))
         LogicError("Ledger::rawErase: key not found");
 }
 
 void
 Ledger::rawErase(uint256 const& key)
 {
-    if (!stateMap_->delItem(key))
+    if (!stateMap_.delItem(key))
         LogicError("Ledger::rawErase: key not found");
 }
 
@@ -574,9 +589,9 @@ Ledger::rawInsert(std::shared_ptr<SLE> const& sle)
 {
     Serializer ss;
     sle->add(ss);
-    if (!stateMap_->addGiveItem(
+    if (!stateMap_.addGiveItem(
             SHAMapNodeType::tnACCOUNT_STATE,
-            std::make_shared<SHAMapItem const>(sle->key(), ss.slice())))
+            make_shamapitem(sle->key(), ss.slice())))
         LogicError("Ledger::rawInsert: key already exists");
 }
 
@@ -585,9 +600,9 @@ Ledger::rawReplace(std::shared_ptr<SLE> const& sle)
 {
     Serializer ss;
     sle->add(ss);
-    if (!stateMap_->updateGiveItem(
+    if (!stateMap_.updateGiveItem(
             SHAMapNodeType::tnACCOUNT_STATE,
-            std::make_shared<SHAMapItem const>(sle->key(), ss.slice())))
+            make_shamapitem(sle->key(), ss.slice())))
         LogicError("Ledger::rawReplace: key not found");
 }
 
@@ -603,9 +618,8 @@ Ledger::rawTxInsert(
     Serializer s(txn->getDataLength() + metaData->getDataLength() + 16);
     s.addVL(txn->peekData());
     s.addVL(metaData->peekData());
-    if (!txMap().addGiveItem(
-            SHAMapNodeType::tnTRANSACTION_MD,
-            std::make_shared<SHAMapItem const>(key, s.slice())))
+    if (!txMap_.addGiveItem(
+            SHAMapNodeType::tnTRANSACTION_MD, make_shamapitem(key, s.slice())))
         LogicError("duplicate_tx: " + to_string(key));
 }
 
@@ -621,9 +635,9 @@ Ledger::rawTxInsertWithHash(
     Serializer s(txn->getDataLength() + metaData->getDataLength() + 16);
     s.addVL(txn->peekData());
     s.addVL(metaData->peekData());
-    auto item = std::make_shared<SHAMapItem const>(key, s.slice());
+    auto item = make_shamapitem(key, s.slice());
     auto hash = sha512Half(HashPrefix::txNode, item->slice(), item->key());
-    if (!txMap().addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, std::move(item)))
+    if (!txMap_.addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, std::move(item)))
         LogicError("duplicate_tx: " + to_string(key));
 
     return hash;
@@ -723,7 +737,7 @@ Ledger::defaultFees(Config const& config)
 std::shared_ptr<SLE>
 Ledger::peek(Keylet const& k) const
 {
-    auto const& value = stateMap_->peekItem(k.key);
+    auto const& value = stateMap_.peekItem(k.key);
     if (!value)
         return nullptr;
     auto sle = std::make_shared<SLE>(SerialIter{value->slice()}, value->key());
@@ -845,8 +859,8 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
     std::vector<SHAMapMissingNode> missingNodes1;
     std::vector<SHAMapMissingNode> missingNodes2;
 
-    if (stateMap_->getHash().isZero() && !info_.accountHash.isZero() &&
-        !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
+    if (stateMap_.getHash().isZero() && !info_.accountHash.isZero() &&
+        !stateMap_.fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
     {
         missingNodes1.emplace_back(
             SHAMapType::STATE, SHAMapHash{info_.accountHash});
@@ -854,9 +868,9 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
     else
     {
         if (parallel)
-            return stateMap_->walkMapParallel(missingNodes1, 32);
+            return stateMap_.walkMapParallel(missingNodes1, 32);
         else
-            stateMap_->walkMap(missingNodes1, 32);
+            stateMap_.walkMap(missingNodes1, 32);
     }
 
     if (!missingNodes1.empty())
@@ -868,15 +882,15 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
         }
     }
 
-    if (txMap_->getHash().isZero() && info_.txHash.isNonZero() &&
-        !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
+    if (txMap_.getHash().isZero() && info_.txHash.isNonZero() &&
+        !txMap_.fetchRoot(SHAMapHash{info_.txHash}, nullptr))
     {
         missingNodes2.emplace_back(
             SHAMapType::TRANSACTION, SHAMapHash{info_.txHash});
     }
     else
     {
-        txMap_->walkMap(missingNodes2, 32);
+        txMap_.walkMap(missingNodes2, 32);
     }
 
     if (!missingNodes2.empty())
@@ -893,9 +907,9 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
 bool
 Ledger::assertSensible(beast::Journal ledgerJ) const
 {
-    if (info_.hash.isNonZero() && info_.accountHash.isNonZero() && stateMap_ &&
-        txMap_ && (info_.accountHash == stateMap_->getHash().as_uint256()) &&
-        (info_.txHash == txMap_->getHash().as_uint256()))
+    if (info_.hash.isNonZero() && info_.accountHash.isNonZero() &&
+        (info_.accountHash == stateMap_.getHash().as_uint256()) &&
+        (info_.txHash == txMap_.getHash().as_uint256()))
     {
         return true;
     }
@@ -1057,15 +1071,14 @@ pendSaveValidated(
         return true;
     }
 
-    JobType const jobType{isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER};
-    char const* const jobName{
-        isCurrent ? "Ledger::pendSave" : "Ledger::pendOldSave"};
-
     // See if we can use the JobQueue.
     if (!isSynchronous &&
-        app.getJobQueue().addJob(jobType, jobName, [&app, ledger, isCurrent]() {
-            saveValidatedLedger(app, ledger, isCurrent);
-        }))
+        app.getJobQueue().addJob(
+            isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER,
+            std::to_string(ledger->seq()),
+            [&app, ledger, isCurrent]() {
+                saveValidatedLedger(app, ledger, isCurrent);
+            }))
     {
         return true;
     }
@@ -1077,15 +1090,15 @@ pendSaveValidated(
 void
 Ledger::unshare() const
 {
-    stateMap_->unshare();
-    txMap_->unshare();
+    stateMap_.unshare();
+    txMap_.unshare();
 }
 
 void
 Ledger::invariants() const
 {
-    stateMap_->invariants();
-    txMap_->invariants();
+    stateMap_.invariants();
+    txMap_.invariants();
 }
 //------------------------------------------------------------------------------
 

src/ripple/app/ledger/Ledger.h

@@ -83,6 +83,10 @@ public:
     Ledger&
     operator=(Ledger const&) = delete;
 
+    Ledger(Ledger&&) = delete;
+    Ledger&
+    operator=(Ledger&&) = delete;
+
     /** Create the Genesis ledger.
 
         The Genesis ledger contains a single account whose
@@ -117,6 +121,13 @@ public:
         Family& family,
         beast::Journal j);
 
+    // used when loading ledgers from catalogue files
+    Ledger(
+        LedgerInfo& info,
+        Config const& config,
+        Family& family,
+        SHAMap const& baseState);
+
     /** Create a new ledger following a previous ledger
 
         The ledger will have the sequence number that
@@ -271,6 +282,12 @@ public:
     void
     setImmutable(bool rehash = true);
 
+    void
+    setCloseFlags(int closeFlags);
+
+    void
+    setDrops(uint64_t drops);
+
     bool
     isImmutable() const
     {
@@ -290,10 +307,10 @@ public:
     void
     setFull() const
     {
-        txMap_->setFull();
-        stateMap_->setFull();
-        txMap_->setLedgerSeq(info_.seq);
-        stateMap_->setLedgerSeq(info_.seq);
+        txMap_.setFull();
+        txMap_.setLedgerSeq(info_.seq);
+        stateMap_.setFull();
+        stateMap_.setLedgerSeq(info_.seq);
     }
 
     void
@@ -305,25 +322,25 @@ public:
     SHAMap const&
     stateMap() const
     {
-        return *stateMap_;
+        return stateMap_;
     }
 
     SHAMap&
     stateMap()
     {
-        return *stateMap_;
+        return stateMap_;
     }
 
     SHAMap const&
     txMap() const
     {
-        return *txMap_;
+        return txMap_;
     }
 
     SHAMap&
     txMap()
     {
-        return *txMap_;
+        return txMap_;
     }
 
     // returns false on error
@@ -401,8 +418,11 @@ private:
 
     bool mImmutable;
 
-    std::shared_ptr<SHAMap> txMap_;
-    std::shared_ptr<SHAMap> stateMap_;
+    // A SHAMap containing the transactions associated with this ledger.
+    SHAMap mutable txMap_;
+
+    // A SHAMap containing the state objects for this ledger.
+    SHAMap mutable stateMap_;
 
     // Protects fee variables
     std::mutex mutable mutex_;

src/ripple/app/ledger/LedgerHistory.cpp

@@ -51,7 +51,9 @@ LedgerHistory::LedgerHistory(
 }
 
 bool
-LedgerHistory::insert(std::shared_ptr<Ledger const> ledger, bool validated)
+LedgerHistory::insert(
+    std::shared_ptr<Ledger const> const& ledger,
+    bool validated)
 {
     if (!ledger->isImmutable())
         LogicError("mutable Ledger in insert");
@@ -72,12 +74,9 @@ LedgerHash
 LedgerHistory::getLedgerHash(LedgerIndex index)
 {
     std::unique_lock sl(m_ledgers_by_hash.peekMutex());
-    auto it = mLedgersByIndex.find(index);
-
-    if (it != mLedgersByIndex.end())
+    if (auto it = mLedgersByIndex.find(index); it != mLedgersByIndex.end())
         return it->second;
-
-    return uint256();
+    return {};
 }
 
 std::shared_ptr<Ledger const>
@@ -167,19 +166,19 @@ log_metadata_difference(
     uint256 const& tx,
     beast::Journal j)
 {
-    auto getMeta = [](ReadView const& ledger,
-                      uint256 const& txID) -> std::shared_ptr<TxMeta> {
-        auto meta = ledger.txRead(txID).second;
-        if (!meta)
-            return {};
-        return std::make_shared<TxMeta>(txID, ledger.seq(), *meta);
+    auto getMeta = [](ReadView const& ledger, uint256 const& txID) {
+        std::optional<TxMeta> ret;
+        if (auto meta = ledger.txRead(txID).second)
+            ret.emplace(txID, ledger.seq(), *meta);
+        return ret;
     };
 
     auto validMetaData = getMeta(validLedger, tx);
     auto builtMetaData = getMeta(builtLedger, tx);
-    assert(validMetaData != nullptr || builtMetaData != nullptr);
 
-    if (validMetaData != nullptr && builtMetaData != nullptr)
+    assert(validMetaData || builtMetaData);
+
+    if (validMetaData && builtMetaData)
     {
         auto const& validNodes = validMetaData->getNodes();
         auto const& builtNodes = builtMetaData->getNodes();
@@ -280,17 +279,21 @@ log_metadata_difference(
                                 << validNodes.getJson(JsonOptions::none);
             }
         }
+
+        return;
     }
-    else if (validMetaData != nullptr)
+
+    if (validMetaData)
     {
         JLOG(j.error()) << "MISMATCH on TX " << tx
-                        << ": Metadata Difference (built has none)\n"
+                        << ": Metadata Difference. Valid=\n"
                         << validMetaData->getJson(JsonOptions::none);
     }
-    else  // builtMetaData != nullptr
+
+    if (builtMetaData)
     {
         JLOG(j.error()) << "MISMATCH on TX " << tx
-                        << ": Metadata Difference (valid has none)\n"
+                        << ": Metadata Difference. Built=\n"
                         << builtMetaData->getJson(JsonOptions::none);
     }
 }

src/ripple/app/ledger/LedgerHistory.h

@@ -44,7 +44,7 @@ public:
         @return `true` if the ledger was already tracked
     */
     bool
-    insert(std::shared_ptr<Ledger const> ledger, bool validated);
+    insert(std::shared_ptr<Ledger const> const& ledger, bool validated);
 
     /** Get the ledgers_by_hash cache hit rate
         @return the hit rate
@@ -70,8 +70,6 @@ public:
     LedgerHash
     getLedgerHash(LedgerIndex ledgerIndex);
 
-    /** Remove stale cache entries
-     */
     void
     sweep()
     {

src/ripple/app/ledger/LedgerMaster.h

@@ -128,7 +128,7 @@ public:
     getEarliestFetch();
 
     bool
-    storeLedger(std::shared_ptr<Ledger const> ledger);
+    storeLedger(std::shared_ptr<Ledger const> ledger, bool pin = false);
 
     void
     setFullLedger(
@@ -152,6 +152,15 @@ public:
     std::string
     getCompleteLedgers();
 
+    std::string
+    getPinnedLedgers();
+
+    RangeSet<std::uint32_t>
+    getCompleteLedgersRangeSet();
+
+    RangeSet<std::uint32_t>
+    getPinnedLedgersRangeSet();
+
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
         The open ledger remains open to handle new transactions
@@ -197,7 +206,10 @@ public:
     getLedgerByHash(uint256 const& hash);
 
     void
-    setLedgerRangePresent(std::uint32_t minV, std::uint32_t maxV);
+    setLedgerRangePresent(
+        std::uint32_t minV,
+        std::uint32_t maxV,
+        bool pin = false /* if true, do not let these leaders be removed */);
 
     std::optional<NetClock::time_point>
     getCloseTimeBySeq(LedgerIndex ledgerIndex);
@@ -370,6 +382,7 @@ private:
 
     std::recursive_mutex mCompleteLock;
     RangeSet<std::uint32_t> mCompleteLedgers;
+    RangeSet<std::uint32_t> mPinnedLedgers;  // Track pinned ledger ranges
 
     // Publish thread is running.
     bool mAdvanceThread{false};

src/ripple/app/ledger/LedgerReplayer.h

@@ -105,7 +105,7 @@ public:
     void
     gotSkipList(
         LedgerInfo const& info,
-        std::shared_ptr<SHAMapItem const> const& data);
+        boost::intrusive_ptr<SHAMapItem const> const& data);
 
     /**
      * Process a ledger delta (extracted from a TMReplayDeltaResponse message)

src/ripple/app/ledger/TransactionMaster.h

@@ -68,7 +68,7 @@ public:
 
     std::shared_ptr<STTx const>
     fetch(
-        std::shared_ptr<SHAMapItem> const& item,
+        boost::intrusive_ptr<SHAMapItem> const& item,
         SHAMapNodeType type,
         std::uint32_t uCommitLedger);
 

src/ripple/app/ledger/impl/LedgerCleaner.cpp

@@ -219,7 +219,7 @@ private:
     run()
     {
         beast::setCurrentThreadName("LedgerCleaner");
-        JLOG(j_.debug()) << "Started";
+        JLOG(j_.debug()) << "Started ledger cleaner";
 
         while (true)
         {
@@ -392,7 +392,8 @@ private:
 
             if (app_.getFeeTrack().isLoadedLocal())
             {
-                JLOG(j_.debug()) << "Waiting for load to subside";
+                JLOG(j_.debug())
+                    << "Ledger Cleaner: Waiting for load to subside";
                 std::this_thread::sleep_for(std::chrono::seconds(5));
                 continue;
             }
@@ -415,13 +416,15 @@ private:
             bool fail = false;
             if (ledgerHash.isZero())
             {
-                JLOG(j_.info())
-                    << "Unable to get hash for ledger " << ledgerIndex;
+                JLOG(j_.warn())
+                    << "Ledger Cleaner: Unable to get hash for ledger "
+                    << ledgerIndex;
                 fail = true;
             }
             else if (!doLedger(ledgerIndex, ledgerHash, doNodes, doTxns))
             {
-                JLOG(j_.info()) << "Failed to process ledger " << ledgerIndex;
+                JLOG(j_.warn()) << "Ledger Cleaner: Failed to process ledger "
+                                << ledgerIndex;
                 fail = true;
             }
 

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -533,11 +533,20 @@ LedgerMaster::fixIndex(LedgerIndex ledgerIndex, LedgerHash const& ledgerHash)
 }
 
 bool
-LedgerMaster::storeLedger(std::shared_ptr<Ledger const> ledger)
+LedgerMaster::storeLedger(std::shared_ptr<Ledger const> ledger, bool pin)
 {
     bool validated = ledger->info().validated;
     // Returns true if we already had the ledger
-    return mLedgerHistory.insert(std::move(ledger), validated);
+    if (!mLedgerHistory.insert(std::move(ledger), validated))
+        return false;
+
+    if (pin)
+    {
+        uint32_t seq = ledger->info().seq;
+        mPinnedLedgers.insert(range(seq, seq));
+        JLOG(m_journal.info()) << "Pinned ledger : " << seq;
+    }
+    return true;
 }
 
 /** Apply held transactions to the open ledger
@@ -595,6 +604,15 @@ void
 LedgerMaster::clearLedger(std::uint32_t seq)
 {
     std::lock_guard sl(mCompleteLock);
+
+    // Don't clear pinned ledgers
+    if (boost::icl::contains(mPinnedLedgers, seq))
+    {
+        JLOG(m_journal.trace())
+            << "Ledger " << seq << " is pinned, not clearing";
+        return;
+    }
+
     mCompleteLedgers.erase(seq);
 }
 
@@ -1714,6 +1732,27 @@ LedgerMaster::getCompleteLedgers()
     return to_string(mCompleteLedgers);
 }
 
+std::string
+LedgerMaster::getPinnedLedgers()
+{
+    std::lock_guard sl(mCompleteLock);
+    return to_string(mPinnedLedgers);
+}
+
+RangeSet<std::uint32_t>
+LedgerMaster::getCompleteLedgersRangeSet()
+{
+    std::lock_guard sl(mCompleteLock);
+    return mCompleteLedgers;
+}
+
+RangeSet<std::uint32_t>
+LedgerMaster::getPinnedLedgersRangeSet()
+{
+    std::lock_guard sl(mCompleteLock);
+    return mPinnedLedgers;
+}
+
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {
@@ -1869,15 +1908,26 @@ LedgerMaster::getLedgerByHash(uint256 const& hash)
 }
 
 void
-LedgerMaster::setLedgerRangePresent(std::uint32_t minV, std::uint32_t maxV)
+LedgerMaster::setLedgerRangePresent(
+    std::uint32_t minV,
+    std::uint32_t maxV,
+    bool pin)
 {
     std::lock_guard sl(mCompleteLock);
     mCompleteLedgers.insert(range(minV, maxV));
+
+    if (pin)
+    {
+        mPinnedLedgers.insert(range(minV, maxV));
+        JLOG(m_journal.info())
+            << "Pinned ledger range: " << minV << " - " << maxV;
+    }
 }
 
 void
 LedgerMaster::sweep()
 {
+    std::lock_guard sl(mCompleteLock);
     mLedgerHistory.sweep();
     fetch_packs_.sweep();
 }
@@ -1892,8 +1942,24 @@ void
 LedgerMaster::clearPriorLedgers(LedgerIndex seq)
 {
     std::lock_guard sl(mCompleteLock);
-    if (seq > 0)
-        mCompleteLedgers.erase(range(0u, seq - 1));
+    if (seq <= 0)
+        return;
+
+    // First, save a copy of the pinned ledgers
+    auto pinnedCopy = mPinnedLedgers;
+
+    // Clear everything before seq
+    RangeSet<std::uint32_t> toClear;
+    toClear.insert(range(0u, seq - 1));
+    for (auto const& interval : toClear)
+        mCompleteLedgers.erase(interval);
+
+    // Re-add the pinned ledgers to ensure they're preserved
+    for (auto const& interval : pinnedCopy)
+        mCompleteLedgers.insert(interval);
+
+    JLOG(m_journal.debug()) << "clearPriorLedgers: after restoration, pinned="
+                            << to_string(mPinnedLedgers);
 }
 
 void

src/ripple/app/ledger/impl/LedgerReplayMsgHandler.cpp

@@ -163,15 +163,15 @@ LedgerReplayMsgHandler::processProofPathResponse(
         JLOG(journal_.debug()) << "Bad message: Cannot deserialize";
         return false;
     }
-    auto item = static_cast<SHAMapLeafNode*>(node.get())->peekItem();
-    if (!item)
+
+    if (auto item = static_cast<SHAMapLeafNode*>(node.get())->peekItem())
     {
-        JLOG(journal_.debug()) << "Bad message: Cannot get ShaMapItem";
-        return false;
+        replayer_.gotSkipList(info, item);
+        return true;
     }
 
-    replayer_.gotSkipList(info, item);
-    return true;
+    JLOG(journal_.debug()) << "Bad message: Cannot get ShaMapItem";
+    return false;
 }
 
 protocol::TMReplayDeltaResponse
@@ -206,9 +206,10 @@ LedgerReplayMsgHandler::processReplayDeltaRequest(
     reply.set_ledgerheader(nData.getDataPtr(), nData.getLength());
     // pack transactions
     auto const& txMap = ledger->txMap();
-    txMap.visitLeaves([&](std::shared_ptr<SHAMapItem const> const& txNode) {
-        reply.add_transaction(txNode->data(), txNode->size());
-    });
+    txMap.visitLeaves(
+        [&](boost::intrusive_ptr<SHAMapItem const> const& txNode) {
+            reply.add_transaction(txNode->data(), txNode->size());
+        });
 
     JLOG(journal_.debug()) << "getReplayDelta for ledger " << ledgerHash
                            << " txMap hash " << txMap.getHash().as_uint256();
@@ -264,10 +265,9 @@ LedgerReplayMsgHandler::processReplayDeltaResponse(
             STObject meta(metaSit, sfMetadata);
             orderedTxns.emplace(meta[sfTransactionIndex], std::move(tx));
 
-            auto item =
-                std::make_shared<SHAMapItem const>(tid, shaMapItemData.slice());
-            if (!item ||
-                !txMap.addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, item))
+            if (!txMap.addGiveItem(
+                    SHAMapNodeType::tnTRANSACTION_MD,
+                    make_shamapitem(tid, shaMapItemData.slice())))
             {
                 JLOG(journal_.debug()) << "Bad message: Cannot deserialize";
                 return false;

src/ripple/app/ledger/impl/LedgerReplayer.cpp

@@ -172,7 +172,7 @@ LedgerReplayer::createDeltas(std::shared_ptr<LedgerReplayTask> task)
 void
 LedgerReplayer::gotSkipList(
     LedgerInfo const& info,
-    std::shared_ptr<SHAMapItem const> const& item)
+    boost::intrusive_ptr<SHAMapItem const> const& item)
 {
     std::shared_ptr<SkipListAcquire> skipList = {};
     {

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -131,14 +131,14 @@ fillJsonTx(
         if (stMeta)
         {
             txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
+
+            // If applicable, insert delivered amount
             if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
-            {
-                // Insert delivered amount
-                auto txMeta = std::make_shared<TxMeta>(
-                    txn->getTransactionID(), fill.ledger.seq(), *stMeta);
                 RPC::insertDeliveredAmount(
-                    txJson[jss::metaData], fill.ledger, txn, *txMeta);
-            }
+                    txJson[jss::metaData],
+                    fill.ledger,
+                    txn,
+                    {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
         }
     }
 

src/ripple/app/ledger/impl/SkipListAcquire.cpp

@@ -137,7 +137,7 @@ SkipListAcquire::pmDowncast()
 void
 SkipListAcquire::processData(
     std::uint32_t ledgerSeq,
-    std::shared_ptr<SHAMapItem const> const& item)
+    boost::intrusive_ptr<SHAMapItem const> const& item)
 {
     assert(ledgerSeq != 0 && item);
     ScopedLockType sl(mtx_);

src/ripple/app/ledger/impl/SkipListAcquire.h

@@ -96,7 +96,7 @@ public:
     void
     processData(
         std::uint32_t ledgerSeq,
-        std::shared_ptr<SHAMapItem const> const& item);
+        boost::intrusive_ptr<SHAMapItem const> const& item);
 
     /**
      * Add a callback that will be called when the skipList is ready or failed.

src/ripple/app/ledger/impl/TransactionMaster.cpp

@@ -111,7 +111,7 @@ TransactionMaster::fetch(
 
 std::shared_ptr<STTx const>
 TransactionMaster::fetch(
-    std::shared_ptr<SHAMapItem> const& item,
+    boost::intrusive_ptr<SHAMapItem> const& item,
     SHAMapNodeType type,
     std::uint32_t uCommitLedger)
 {

src/ripple/app/main/Application.cpp

@@ -37,6 +37,7 @@
 #include <ripple/app/main/NodeStoreScheduler.h>
 #include <ripple/app/main/Tuning.h>
 #include <ripple/app/misc/AmendmentTable.h>
+#include <ripple/app/misc/DatagramMonitor.h>
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
 #include <ripple/app/misc/NetworkOPs.h>
@@ -167,6 +168,8 @@ public:
     std::unique_ptr<Logs> logs_;
     std::unique_ptr<TimeKeeper> timeKeeper_;
 
+    std::unique_ptr<DatagramMonitor> datagram_monitor_;
+
     std::uint64_t const instanceCookie_;
 
     beast::Journal m_journal;
@@ -1523,6 +1526,14 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
     if (reportingETL_)
         reportingETL_->start();
 
+    // Datagram monitor if applicable
+    if (!config_->standalone() && !config_->DATAGRAM_MONITOR.empty())
+    {
+        datagram_monitor_ = std::make_unique<DatagramMonitor>(*this);
+        if (datagram_monitor_)
+            datagram_monitor_->start();
+    }
+
     return true;
 }
 

src/ripple/app/main/Application.h

@@ -19,7 +19,6 @@
 
 #ifndef RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED
 #define RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED
-
 #include <ripple/basics/TaggedCache.h>
 #include <ripple/beast/utility/PropertyStream.h>
 #include <ripple/core/Config.h>

src/ripple/app/main/Main.cpp

@@ -124,14 +124,12 @@ printHelp(const po::options_description& desc)
         << systemName() << "d [options] <command> <params>\n"
         << desc << std::endl
         << "Commands: \n"
-           "     account_currencies <account> [<ledger>] [strict]\n"
-           "     account_info <account>|<seed>|<pass_phrase>|<key> [<ledger>] "
-           "[strict]\n"
+           "     account_currencies <account> [<ledger>]\n"
+           "     account_info <account>|<key> [<ledger>]\n"
            "     account_lines <account> <account>|\"\" [<ledger>]\n"
            "     account_channels <account> <account>|\"\" [<ledger>]\n"
-           "     account_objects <account> [<ledger>] [strict]\n"
-           "     account_offers <account>|<account_public_key> [<ledger>] "
-           "[strict]\n"
+           "     account_objects <account> [<ledger>]\n"
+           "     account_offers <account>|<account_public_key> [<ledger>]\n"
            "     account_tx accountID [ledger_index_min [ledger_index_max "
            "[limit "
            "]]] [binary]\n"

src/ripple/app/misc/AmendmentTable.h

@@ -172,8 +172,7 @@ public:
 
             initialPosition->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(
-                    amendTx.getTransactionID(), s.slice()));
+                make_shamapitem(amendTx.getTransactionID(), s.slice()));
         }
     }
 };

src/ripple/app/misc/FeeVoteImpl.cpp

@@ -326,7 +326,7 @@ FeeVoteImpl::doVoting(
 
         if (!initialPosition->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(journal_.warn()) << "Ledger already had fee change";
         }

src/ripple/app/misc/NegativeUNLVote.cpp

@@ -22,6 +22,7 @@
 #include <ripple/app/main/Application.h>
 #include <ripple/app/misc/NegativeUNLVote.h>
 #include <ripple/json/to_string.h>
+#include <ripple/shamap/SHAMapItem.h>
 
 namespace ripple {
 
@@ -147,7 +148,7 @@ NegativeUNLVote::addReportingTx(
         repUnlTx.add(s);
         if (!initalSet->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(j_.warn()) << "R-UNL: ledger seq=" << seq
                             << ", add ttUNL_REPORT tx failed";
@@ -202,7 +203,7 @@ NegativeUNLVote::addImportVLTx(
         repUnlTx.add(s);
         if (!initalSet->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(j_.warn()) << "R-UNL: ledger seq=" << seq
                             << ", add ttUNL_REPORT tx failed (import_vl_key)";
@@ -231,12 +232,11 @@ NegativeUNLVote::addTx(
         obj.setFieldVL(sfUNLModifyValidator, vp.slice());
     });
 
-    uint256 txID = negUnlTx.getTransactionID();
     Serializer s;
     negUnlTx.add(s);
     if (!initialSet->addGiveItem(
             SHAMapNodeType::tnTRANSACTION_NM,
-            std::make_shared<SHAMapItem>(txID, s.slice())))
+            make_shamapitem(negUnlTx.getTransactionID(), s.slice())))
     {
         JLOG(j_.warn()) << "N-UNL: ledger seq=" << seq
                         << ", add ttUNL_MODIFY tx failed";
@@ -244,8 +244,8 @@ NegativeUNLVote::addTx(
     else
     {
         JLOG(j_.debug()) << "N-UNL: ledger seq=" << seq
-                         << ", add a ttUNL_MODIFY Tx with txID: " << txID
-                         << ", the validator to "
+                         << ", add a ttUNL_MODIFY Tx with txID: "
+                         << negUnlTx.getTransactionID() << ", the validator to "
                          << (modify == ToDisable ? "disable: " : "re-enable: ")
                          << vp;
     }

src/ripple/app/misc/NetworkOPs.cpp

@@ -33,6 +33,7 @@
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
 #include <ripple/app/misc/NetworkOPs.h>
+#include <ripple/app/misc/StateAccounting.h>
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/app/misc/ValidatorKeys.h>
@@ -67,9 +68,9 @@
 #include <ripple/rpc/CTID.h>
 #include <ripple/rpc/DeliveredAmount.h>
 #include <ripple/rpc/impl/RPCHelpers.h>
+#include <ripple/rpc/impl/UDPInfoSub.h>
 #include <boost/asio/ip/host_name.hpp>
 #include <boost/asio/steady_timer.hpp>
-
 #include <exception>
 #include <mutex>
 #include <set>
@@ -116,81 +117,6 @@ class NetworkOPsImp final : public NetworkOPs
         running,
     };
 
-    static std::array<char const*, 5> const states_;
-
-    /**
-     * State accounting records two attributes for each possible server state:
-     * 1) Amount of time spent in each state (in microseconds). This value is
-     *    updated upon each state transition.
-     * 2) Number of transitions to each state.
-     *
-     * This data can be polled through server_info and represented by
-     * monitoring systems similarly to how bandwidth, CPU, and other
-     * counter-based metrics are managed.
-     *
-     * State accounting is more accurate than periodic sampling of server
-     * state. With periodic sampling, it is very likely that state transitions
-     * are missed, and accuracy of time spent in each state is very rough.
-     */
-    class StateAccounting
-    {
-        struct Counters
-        {
-            explicit Counters() = default;
-
-            std::uint64_t transitions = 0;
-            std::chrono::microseconds dur = std::chrono::microseconds(0);
-        };
-
-        OperatingMode mode_ = OperatingMode::DISCONNECTED;
-        std::array<Counters, 5> counters_;
-        mutable std::mutex mutex_;
-        std::chrono::steady_clock::time_point start_ =
-            std::chrono::steady_clock::now();
-        std::chrono::steady_clock::time_point const processStart_ = start_;
-        std::uint64_t initialSyncUs_{0};
-        static std::array<Json::StaticString const, 5> const states_;
-
-    public:
-        explicit StateAccounting()
-        {
-            counters_[static_cast<std::size_t>(OperatingMode::DISCONNECTED)]
-                .transitions = 1;
-        }
-
-        /**
-         * Record state transition. Update duration spent in previous
-         * state.
-         *
-         * @param om New state.
-         */
-        void
-        mode(OperatingMode om);
-
-        /**
-         * Output state counters in JSON format.
-         *
-         * @obj Json object to which to add state accounting data.
-         */
-        void
-        json(Json::Value& obj) const;
-
-        struct CounterData
-        {
-            decltype(counters_) counters;
-            decltype(mode_) mode;
-            decltype(start_) start;
-            decltype(initialSyncUs_) initialSyncUs;
-        };
-
-        CounterData
-        getCounterData() const
-        {
-            std::lock_guard lock(mutex_);
-            return {counters_, mode_, start_, initialSyncUs_};
-        }
-    };
-
     //! Server fees published on `server` subscription
     struct ServerFeeSummary
     {
@@ -272,6 +198,9 @@ public:
     std::string
     strOperatingMode(bool const admin = false) const override;
 
+    StateAccounting::CounterData
+    getStateAccountingData();
+
     //
     // Transaction operations.
     //
@@ -776,11 +705,17 @@ private:
     DispatchState mDispatchState = DispatchState::none;
     std::vector<TransactionStatus> mTransactions;
 
-    StateAccounting accounting_{};
+    StateAccounting accounting_;
 
     std::set<uint256> pendingValidations_;
     std::mutex validationsMutex_;
 
+    RCLConsensus&
+    getConsensus();
+
+    LedgerMaster&
+    getLedgerMaster();
+
 private:
     struct Stats
     {
@@ -843,19 +778,6 @@ private:
 
 //------------------------------------------------------------------------------
 
-static std::array<char const*, 5> const stateNames{
-    {"disconnected", "connected", "syncing", "tracking", "full"}};
-
-std::array<char const*, 5> const NetworkOPsImp::states_ = stateNames;
-
-std::array<Json::StaticString const, 5> const
-    NetworkOPsImp::StateAccounting::states_ = {
-        {Json::StaticString(stateNames[0]),
-         Json::StaticString(stateNames[1]),
-         Json::StaticString(stateNames[2]),
-         Json::StaticString(stateNames[3]),
-         Json::StaticString(stateNames[4])}};
-
 static auto const genesisAccountId = calcAccountID(
     generateKeyPair(KeyType::secp256k1, generateSeed("masterpassphrase"))
         .first);
@@ -1130,7 +1052,7 @@ NetworkOPsImp::strOperatingMode(OperatingMode const mode, bool const admin)
         }
     }
 
-    return states_[static_cast<std::size_t>(mode)];
+    return {StateAccounting::states_[static_cast<std::size_t>(mode)].c_str()};
 }
 
 void
@@ -2396,6 +2318,19 @@ NetworkOPsImp::getConsensusInfo()
     return mConsensus.getJson(true);
 }
 
+// RHTODO: not threadsafe?
+RCLConsensus&
+NetworkOPsImp::getConsensus()
+{
+    return mConsensus;
+}
+
+LedgerMaster&
+NetworkOPsImp::getLedgerMaster()
+{
+    return m_ledgerMaster;
+}
+
 Json::Value
 NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
 {
@@ -2558,6 +2493,8 @@ NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
         toBase58(TokenType::NodePublic, app_.nodeIdentity().first);
 
     info[jss::complete_ledgers] = app_.getLedgerMaster().getCompleteLedgers();
+    info[jss::complete_ledgers_pinned] =
+        app_.getLedgerMaster().getPinnedLedgers();
 
     if (amendmentBlocked_)
         info[jss::amendment_blocked] = true;
@@ -4193,6 +4130,12 @@ NetworkOPsImp::stateAccounting(Json::Value& obj)
     accounting_.json(obj);
 }
 
+StateAccounting::CounterData
+NetworkOPsImp::getStateAccountingData()
+{
+    return accounting_.getCounterData();
+}
+
 // <-- bool: true=erased, false=was not there
 bool
 NetworkOPsImp::unsubValidations(std::uint64_t uSeq)
@@ -4663,50 +4606,6 @@ NetworkOPsImp::collect_metrics()
         counters[static_cast<std::size_t>(OperatingMode::FULL)].transitions);
 }
 
-void
-NetworkOPsImp::StateAccounting::mode(OperatingMode om)
-{
-    auto now = std::chrono::steady_clock::now();
-
-    std::lock_guard lock(mutex_);
-    ++counters_[static_cast<std::size_t>(om)].transitions;
-    if (om == OperatingMode::FULL &&
-        counters_[static_cast<std::size_t>(om)].transitions == 1)
-    {
-        initialSyncUs_ = std::chrono::duration_cast<std::chrono::microseconds>(
-                             now - processStart_)
-                             .count();
-    }
-    counters_[static_cast<std::size_t>(mode_)].dur +=
-        std::chrono::duration_cast<std::chrono::microseconds>(now - start_);
-
-    mode_ = om;
-    start_ = now;
-}
-
-void
-NetworkOPsImp::StateAccounting::json(Json::Value& obj) const
-{
-    auto [counters, mode, start, initialSync] = getCounterData();
-    auto const current = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::steady_clock::now() - start);
-    counters[static_cast<std::size_t>(mode)].dur += current;
-
-    obj[jss::state_accounting] = Json::objectValue;
-    for (std::size_t i = static_cast<std::size_t>(OperatingMode::DISCONNECTED);
-         i <= static_cast<std::size_t>(OperatingMode::FULL);
-         ++i)
-    {
-        obj[jss::state_accounting][states_[i]] = Json::objectValue;
-        auto& state = obj[jss::state_accounting][states_[i]];
-        state[jss::transitions] = std::to_string(counters[i].transitions);
-        state[jss::duration_us] = std::to_string(counters[i].dur.count());
-    }
-    obj[jss::server_state_duration_us] = std::to_string(current.count());
-    if (initialSync)
-        obj[jss::initial_sync_duration_us] = std::to_string(initialSync);
-}
-
 //------------------------------------------------------------------------------
 
 std::unique_ptr<NetworkOPs>

src/ripple/app/misc/NetworkOPs.h

@@ -20,8 +20,10 @@
 #ifndef RIPPLE_APP_MISC_NETWORKOPS_H_INCLUDED
 #define RIPPLE_APP_MISC_NETWORKOPS_H_INCLUDED
 
+#include <ripple/app/consensus/RCLConsensus.h>
 #include <ripple/app/consensus/RCLCxPeerPos.h>
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/misc/StateAccounting.h>
 #include <ripple/core/JobQueue.h>
 #include <ripple/ledger/ReadView.h>
 #include <ripple/net/InfoSub.h>
@@ -42,35 +44,6 @@ class LedgerMaster;
 class Transaction;
 class ValidatorKeys;
 
-// This is the primary interface into the "client" portion of the program.
-// Code that wants to do normal operations on the network such as
-// creating and monitoring accounts, creating transactions, and so on
-// should use this interface. The RPC code will primarily be a light wrapper
-// over this code.
-//
-// Eventually, it will check the node's operating mode (synched, unsynched,
-// etectera) and defer to the correct means of processing. The current
-// code assumes this node is synched (and will continue to do so until
-// there's a functional network.
-//
-
-/** Specifies the mode under which the server believes it's operating.
-
-    This has implications about how the server processes transactions and
-    how it responds to requests (e.g. account balance request).
-
-    @note Other code relies on the numerical values of these constants; do
-          not change them without verifying each use and ensuring that it is
-          not a breaking change.
-*/
-enum class OperatingMode {
-    DISCONNECTED = 0,  //!< not ready to process requests
-    CONNECTED = 1,     //!< convinced we are talking to the network
-    SYNCING = 2,       //!< fallen slightly behind
-    TRACKING = 3,      //!< convinced we agree with the network
-    FULL = 4           //!< we have the ledger and can even validate
-};
-
 /** Provides server functionality for clients.
 
     Clients include backend applications, local commands, and connected
@@ -221,6 +194,13 @@ public:
 
     virtual Json::Value
     getConsensusInfo() = 0;
+
+    virtual RCLConsensus&
+    getConsensus() = 0;
+
+    virtual LedgerMaster&
+    getLedgerMaster() = 0;
+
     virtual Json::Value
     getServerInfo(bool human, bool admin, bool counters) = 0;
     virtual void
@@ -228,6 +208,9 @@ public:
     virtual Json::Value
     getLedgerFetchInfo() = 0;
 
+    virtual StateAccounting::CounterData
+    getStateAccountingData() = 0;
+
     /** Accepts the current transaction tree, return the new ledger's sequence
 
         This API is only used via RPC with the server in STANDALONE mode and

src/ripple/app/misc/SHAMapStoreImp.cpp

@@ -162,7 +162,8 @@ SHAMapStoreImp::SHAMapStoreImp(
         }
 
         state_db_.init(config, dbName_);
-        dbPaths();
+        if (!config.mem_backend())
+            dbPaths();
     }
 }
 
@@ -195,6 +196,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
                 "online_delete info from config");
         }
         SavedState state = state_db_.getState();
+
         auto writableBackend = makeBackendRotating(state.writableDb);
         auto archiveBackend = makeBackendRotating(state.archiveDb);
         if (!state.writableDb.size())
@@ -207,6 +209,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
         // Create NodeStore with two backends to allow online deletion of
         // data
         auto dbr = std::make_unique<NodeStore::DatabaseRotatingImp>(
+            app_,
             scheduler_,
             readThreads,
             std::move(writableBackend),
@@ -293,6 +296,8 @@ SHAMapStoreImp::run()
     fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache(0));
     treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache(0));
 
+    bool const isMem = app_.config().mem_backend();
+
     if (advisoryDelete_)
         canDelete_ = state_db_.getCanDelete();
 
@@ -351,7 +356,7 @@ SHAMapStoreImp::run()
         // will delete up to (not including) lastRotated
         if (readyToRotate && !waitForImport)
         {
-            JLOG(journal_.warn())
+            JLOG(journal_.debug())
                 << "rotating  validatedSeq " << validatedSeq << " lastRotated "
                 << lastRotated << " deleteInterval " << deleteInterval_
                 << " canDelete_ " << canDelete_ << " state "
@@ -395,7 +400,7 @@ SHAMapStoreImp::run()
             // Only log if we completed without a "health" abort
             JLOG(journal_.debug()) << validatedSeq << " freshened caches";
 
-            JLOG(journal_.trace()) << "Making a new backend";
+            JLOG(journal_.debug()) << "Making a new backend";
             auto newBackend = makeBackendRotating();
             JLOG(journal_.debug())
                 << validatedSeq << " new backend " << newBackend->getName();
@@ -640,6 +645,33 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
     if (!db)
         Throw<std::runtime_error>("Failed to get relational database");
 
+    if (app_.config().useTxTables())
+    {
+        clearSql(
+            lastRotated,
+            "Transactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+
+        clearSql(
+            lastRotated,
+            "AccountTransactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getAccountTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteAccountTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+    }
+
     clearSql(
         lastRotated,
         "Ledgers",
@@ -647,33 +679,6 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
         [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); });
     if (healthWait() == stopping)
         return;
-
-    if (!app_.config().useTxTables())
-        return;
-
-    clearSql(
-        lastRotated,
-        "Transactions",
-        [&db]() -> std::optional<LedgerIndex> {
-            return db->getTransactionsMinLedgerSeq();
-        },
-        [&db](LedgerIndex min) -> void {
-            db->deleteTransactionsBeforeLedgerSeq(min);
-        });
-    if (healthWait() == stopping)
-        return;
-
-    clearSql(
-        lastRotated,
-        "AccountTransactions",
-        [&db]() -> std::optional<LedgerIndex> {
-            return db->getAccountTransactionsMinLedgerSeq();
-        },
-        [&db](LedgerIndex min) -> void {
-            db->deleteAccountTransactionsBeforeLedgerSeq(min);
-        });
-    if (healthWait() == stopping)
-        return;
 }
 
 SHAMapStoreImp::HealthResult

src/ripple/app/misc/StateAccounting.cpp

@@ -0,0 +1,49 @@
+#include <ripple/app/misc/StateAccounting.h>
+
+namespace ripple {
+
+void
+StateAccounting::mode(OperatingMode om)
+{
+    std::lock_guard lock(mutex_);
+    auto now = std::chrono::steady_clock::now();
+
+    ++counters_[static_cast<std::size_t>(om)].transitions;
+    if (om == OperatingMode::FULL &&
+        counters_[static_cast<std::size_t>(om)].transitions == 1)
+    {
+        initialSyncUs_ = std::chrono::duration_cast<std::chrono::microseconds>(
+                             now - processStart_)
+                             .count();
+    }
+    counters_[static_cast<std::size_t>(mode_)].dur +=
+        std::chrono::duration_cast<std::chrono::microseconds>(now - start_);
+
+    mode_ = om;
+    start_ = now;
+}
+
+void
+StateAccounting::json(Json::Value& obj)
+{
+    auto [counters, mode, start, initialSync] = getCounterData();
+    auto const current = std::chrono::duration_cast<std::chrono::microseconds>(
+        std::chrono::steady_clock::now() - start);
+    counters[static_cast<std::size_t>(mode)].dur += current;
+
+    obj[jss::state_accounting] = Json::objectValue;
+    for (std::size_t i = static_cast<std::size_t>(OperatingMode::DISCONNECTED);
+         i <= static_cast<std::size_t>(OperatingMode::FULL);
+         ++i)
+    {
+        obj[jss::state_accounting][states_[i]] = Json::objectValue;
+        auto& state = obj[jss::state_accounting][states_[i]];
+        state[jss::transitions] = std::to_string(counters[i].transitions);
+        state[jss::duration_us] = std::to_string(counters[i].dur.count());
+    }
+    obj[jss::server_state_duration_us] = std::to_string(current.count());
+    if (initialSync)
+        obj[jss::initial_sync_duration_us] = std::to_string(initialSync);
+}
+
+}  // namespace ripple

src/ripple/app/misc/StateAccounting.h

@@ -0,0 +1,99 @@
+#ifndef RIPPLE_APP_MAIN_STATEACCOUNTING_H_INCLUDED
+#define RIPPLE_APP_MAIN_STATEACCOUNTING_H_INCLUDED
+
+#include <ripple/basics/chrono.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/jss.h>
+#include <array>
+#include <mutex>
+
+namespace ripple {
+
+// This is the primary interface into the "client" portion of the program.
+// Code that wants to do normal operations on the network such as
+// creating and monitoring accounts, creating transactions, and so on
+// should use this interface. The RPC code will primarily be a light wrapper
+// over this code.
+//
+// Eventually, it will check the node's operating mode (synched, unsynched,
+// etectera) and defer to the correct means of processing. The current
+// code assumes this node is synched (and will continue to do so until
+// there's a functional network.
+//
+
+/** Specifies the mode under which the server believes it's operating.
+
+    This has implications about how the server processes transactions and
+    how it responds to requests (e.g. account balance request).
+
+    @note Other code relies on the numerical values of these constants; do
+          not change them without verifying each use and ensuring that it is
+          not a breaking change.
+*/
+enum class OperatingMode {
+    DISCONNECTED = 0,  //!< not ready to process requests
+    CONNECTED = 1,     //!< convinced we are talking to the network
+    SYNCING = 2,       //!< fallen slightly behind
+    TRACKING = 3,      //!< convinced we agree with the network
+    FULL = 4           //!< we have the ledger and can even validate
+};
+
+class StateAccounting
+{
+public:
+    constexpr static std::array<Json::StaticString const, 5> const states_ = {
+        {Json::StaticString("disconnected"),
+         Json::StaticString("connected"),
+         Json::StaticString("syncing"),
+         Json::StaticString("tracking"),
+         Json::StaticString("full")}};
+
+    struct Counters
+    {
+        explicit Counters() = default;
+
+        std::uint64_t transitions = 0;
+        std::chrono::microseconds dur = std::chrono::microseconds(0);
+    };
+
+private:
+    OperatingMode mode_ = OperatingMode::DISCONNECTED;
+    std::array<Counters, 5> counters_;
+    mutable std::mutex mutex_;
+    std::chrono::steady_clock::time_point start_ =
+        std::chrono::steady_clock::now();
+    std::chrono::steady_clock::time_point const processStart_ = start_;
+    std::uint64_t initialSyncUs_{0};
+
+public:
+    explicit StateAccounting()
+    {
+        counters_[static_cast<std::size_t>(OperatingMode::DISCONNECTED)]
+            .transitions = 1;
+    }
+
+    //! Record state transition. Update duration spent in previous state.
+    void
+    mode(OperatingMode om);
+
+    //! Output state counters in JSON format.
+    void
+    json(Json::Value& obj);
+
+    using CounterData = std::tuple<
+        decltype(counters_),
+        decltype(mode_),
+        decltype(start_),
+        decltype(initialSyncUs_)>;
+
+    CounterData
+    getCounterData()
+    {
+        return {counters_, mode_, start_, initialSyncUs_};
+    }
+};
+
+}  // namespace ripple
+
+#endif

src/ripple/app/misc/impl/ValidatorList.cpp

@@ -1710,6 +1710,15 @@ ValidatorList::calculateQuorum(
     std::size_t effectiveUnlSize,
     std::size_t seenSize)
 {
+    // Use quorum if specified via command line.
+    if (minimumQuorum_ > 0)
+    {
+        JLOG(j_.warn()) << "Using potentially unsafe quorum of "
+                        << *minimumQuorum_
+                        << " as specified on the command line";
+        return *minimumQuorum_;
+    }
+
     // Do not use achievable quorum until lists from all configured
     // publishers are available
     for (auto const& list : publisherLists_)
@@ -1752,21 +1761,8 @@ ValidatorList::calculateQuorum(
     // Note that the negative UNL protocol introduced the
     // AbsoluteMinimumQuorum which is 60% of the original UNL size. The
     // effective quorum should not be lower than it.
-    auto quorum = static_cast<std::size_t>(std::max(
+    return static_cast<std::size_t>(std::max(
         std::ceil(effectiveUnlSize * 0.8f), std::ceil(unlSize * 0.6f)));
-
-    // Use lower quorum specified via command line if the normal quorum
-    // appears unreachable based on the number of recently received
-    // validations.
-    if (minimumQuorum_)  // < quorum && seenSize < quorum)
-    {
-        quorum = *minimumQuorum_;
-
-        JLOG(j_.warn()) << "Using unsafe quorum of " << quorum
-                        << " as specified in the command line";
-    }
-
-    return quorum;
 }
 
 TrustChanges

src/ripple/app/rdb/backend/FlatmapDatabase.h

@@ -0,0 +1,853 @@
+#ifndef RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED
+#define RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED
+
+#include <ripple/app/ledger/AcceptedLedger.h>
+#include <ripple/app/ledger/LedgerMaster.h>
+#include <ripple/app/ledger/TransactionMaster.h>
+#include <ripple/app/rdb/backend/SQLiteDatabase.h>
+#include <algorithm>
+#include <map>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <vector>
+
+#include <boost/unordered/concurrent_flat_map.hpp>
+
+namespace ripple {
+
+struct base_uint_hasher
+{
+    using result_type = std::size_t;
+
+    result_type
+    operator()(base_uint<256> const& value) const
+    {
+        return hardened_hash<>{}(value);
+    }
+
+    result_type
+    operator()(AccountID const& value) const
+    {
+        return hardened_hash<>{}(value);
+    }
+};
+
+class FlatmapDatabase : public SQLiteDatabase
+{
+private:
+    struct LedgerData
+    {
+        LedgerInfo info;
+        boost::unordered::
+            concurrent_flat_map<uint256, AccountTx, base_uint_hasher>
+                transactions;
+    };
+
+    struct AccountTxData
+    {
+        boost::unordered::
+            concurrent_flat_map<std::pair<uint32_t, uint32_t>, AccountTx>
+                transactions;
+    };
+
+    Application& app_;
+    Config const& config_;
+    JobQueue& jobQueue_;
+
+    boost::unordered::concurrent_flat_map<LedgerIndex, LedgerData> ledgers_;
+    boost::unordered::
+        concurrent_flat_map<uint256, LedgerIndex, base_uint_hasher>
+            ledgerHashToSeq_;
+    boost::unordered::concurrent_flat_map<uint256, AccountTx, base_uint_hasher>
+        transactionMap_;
+    boost::unordered::
+        concurrent_flat_map<AccountID, AccountTxData, base_uint_hasher>
+            accountTxMap_;
+
+public:
+    FlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue)
+        : app_(app), config_(config), jobQueue_(jobQueue)
+    {
+    }
+
+    std::optional<LedgerIndex>
+    getMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        ledgers_.visit_all([&minSeq](auto const& pair) {
+            if (!minSeq || pair.first < *minSeq)
+            {
+                minSeq = pair.first;
+            }
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getTransactionsMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        transactionMap_.visit_all([&minSeq](auto const& pair) {
+            LedgerIndex seq = pair.second.second->getLgrSeq();
+            if (!minSeq || seq < *minSeq)
+            {
+                minSeq = seq;
+            }
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getAccountTransactionsMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        accountTxMap_.visit_all([&minSeq](auto const& pair) {
+            pair.second.transactions.visit_all([&minSeq](auto const& tx) {
+                if (!minSeq || tx.first.first < *minSeq)
+                {
+                    minSeq = tx.first.first;
+                }
+            });
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getMaxLedgerSeq() override
+    {
+        std::optional<LedgerIndex> maxSeq;
+        ledgers_.visit_all([&maxSeq](auto const& pair) {
+            if (!maxSeq || pair.first > *maxSeq)
+            {
+                maxSeq = pair.first;
+            }
+        });
+        return maxSeq;
+    }
+    void
+    deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.visit(ledgerSeq, [this](auto& item) {
+            item.second.transactions.visit_all([this](auto const& txPair) {
+                transactionMap_.erase(txPair.first);
+            });
+            item.second.transactions.clear();
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first == ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.erase_if([this, ledgerSeq](auto const& item) {
+            if (item.first < ledgerSeq)
+            {
+                item.second.transactions.visit_all([this](auto const& txPair) {
+                    transactionMap_.erase(txPair.first);
+                });
+                ledgerHashToSeq_.erase(item.second.info.hash);
+                return true;
+            }
+            return false;
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.visit_all([this, ledgerSeq](auto& item) {
+            if (item.first < ledgerSeq)
+            {
+                item.second.transactions.visit_all([this](auto const& txPair) {
+                    transactionMap_.erase(txPair.first);
+                });
+                item.second.transactions.clear();
+            }
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+    std::size_t
+    getTransactionCount() override
+    {
+        return transactionMap_.size();
+    }
+
+    std::size_t
+    getAccountTransactionCount() override
+    {
+        std::size_t count = 0;
+        accountTxMap_.visit_all([&count](auto const& item) {
+            count += item.second.transactions.size();
+        });
+        return count;
+    }
+
+    CountMinMax
+    getLedgerCountMinMax() override
+    {
+        CountMinMax result{0, 0, 0};
+        ledgers_.visit_all([&result](auto const& item) {
+            result.numberOfRows++;
+            if (result.minLedgerSequence == 0 ||
+                item.first < result.minLedgerSequence)
+            {
+                result.minLedgerSequence = item.first;
+            }
+            if (item.first > result.maxLedgerSequence)
+            {
+                result.maxLedgerSequence = item.first;
+            }
+        });
+        return result;
+    }
+
+    bool
+    saveValidatedLedger(
+        std::shared_ptr<Ledger const> const& ledger,
+        bool current) override
+    {
+        try
+        {
+            LedgerData ledgerData;
+            ledgerData.info = ledger->info();
+
+            auto aLedger = std::make_shared<AcceptedLedger>(ledger, app_);
+            for (auto const& acceptedLedgerTx : *aLedger)
+            {
+                auto const& txn = acceptedLedgerTx->getTxn();
+                auto const& meta = acceptedLedgerTx->getMeta();
+                auto const& id = txn->getTransactionID();
+
+                std::string reason;
+                auto accTx = std::make_pair(
+                    std::make_shared<ripple::Transaction>(txn, reason, app_),
+                    std::make_shared<ripple::TxMeta>(meta));
+
+                ledgerData.transactions.emplace(id, accTx);
+                transactionMap_.emplace(id, accTx);
+
+                for (auto const& account : meta.getAffectedAccounts())
+                {
+                    accountTxMap_.visit(account, [&](auto& data) {
+                        data.second.transactions.emplace(
+                            std::make_pair(
+                                ledger->info().seq,
+                                acceptedLedgerTx->getTxnSeq()),
+                            accTx);
+                    });
+                }
+            }
+
+            ledgers_.emplace(ledger->info().seq, std::move(ledgerData));
+            ledgerHashToSeq_.emplace(ledger->info().hash, ledger->info().seq);
+
+            if (current)
+            {
+                auto const cutoffSeq =
+                    ledger->info().seq > app_.config().LEDGER_HISTORY
+                    ? ledger->info().seq - app_.config().LEDGER_HISTORY
+                    : 0;
+
+                if (cutoffSeq > 0)
+                {
+                    const std::size_t BATCH_SIZE = 128;
+                    std::size_t deleted = 0;
+
+                    ledgers_.erase_if([&](auto const& item) {
+                        if (deleted >= BATCH_SIZE)
+                            return false;
+
+                        if (item.first < cutoffSeq)
+                        {
+                            item.second.transactions.visit_all(
+                                [this](auto const& txPair) {
+                                    transactionMap_.erase(txPair.first);
+                                });
+                            ledgerHashToSeq_.erase(item.second.info.hash);
+                            deleted++;
+                            return true;
+                        }
+                        return false;
+                    });
+
+                    if (deleted > 0)
+                    {
+                        accountTxMap_.visit_all([cutoffSeq](auto& item) {
+                            item.second.transactions.erase_if(
+                                [cutoffSeq](auto const& tx) {
+                                    return tx.first.first < cutoffSeq;
+                                });
+                        });
+                    }
+
+                    app_.getLedgerMaster().clearPriorLedgers(cutoffSeq);
+                }
+            }
+
+            return true;
+        }
+        catch (std::exception const&)
+        {
+            deleteTransactionByLedgerSeq(ledger->info().seq);
+            return false;
+        }
+    }
+
+    std::optional<LedgerInfo>
+    getLedgerInfoByIndex(LedgerIndex ledgerSeq) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit(ledgerSeq, [&result](auto const& item) {
+            result = item.second.info;
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getNewestLedgerInfo() override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&result](auto const& item) {
+            if (!result || item.second.info.seq > result->seq)
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= ledgerFirstIndex &&
+                (!result || item.first < result->seq))
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= ledgerFirstIndex &&
+                (!result || item.first > result->seq))
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLedgerInfoByHash(uint256 const& ledgerHash) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgerHashToSeq_.visit(ledgerHash, [this, &result](auto const& item) {
+            ledgers_.visit(item.second, [&result](auto const& item) {
+                result = item.second.info;
+            });
+        });
+        return result;
+    }
+    uint256
+    getHashByIndex(LedgerIndex ledgerIndex) override
+    {
+        uint256 result;
+        ledgers_.visit(ledgerIndex, [&result](auto const& item) {
+            result = item.second.info.hash;
+        });
+        return result;
+    }
+
+    std::optional<LedgerHashPair>
+    getHashesByIndex(LedgerIndex ledgerIndex) override
+    {
+        std::optional<LedgerHashPair> result;
+        ledgers_.visit(ledgerIndex, [&result](auto const& item) {
+            result = LedgerHashPair{
+                item.second.info.hash, item.second.info.parentHash};
+        });
+        return result;
+    }
+
+    std::map<LedgerIndex, LedgerHashPair>
+    getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override
+    {
+        std::map<LedgerIndex, LedgerHashPair> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= minSeq && item.first <= maxSeq)
+            {
+                result[item.first] = LedgerHashPair{
+                    item.second.info.hash, item.second.info.parentHash};
+            }
+        });
+        return result;
+    }
+
+    std::variant<AccountTx, TxSearched>
+    getTransaction(
+        uint256 const& id,
+        std::optional<ClosedInterval<std::uint32_t>> const& range,
+        error_code_i& ec) override
+    {
+        std::variant<AccountTx, TxSearched> result = TxSearched::unknown;
+        transactionMap_.visit(id, [&](auto const& item) {
+            auto const& tx = item.second;
+            if (!range ||
+                (range->lower() <= tx.second->getLgrSeq() &&
+                 tx.second->getLgrSeq() <= range->upper()))
+            {
+                result = tx;
+            }
+            else
+            {
+                result = TxSearched::all;
+            }
+        });
+        return result;
+    }
+
+    bool
+    ledgerDbHasSpace(Config const& config) override
+    {
+        return true;  // In-memory database always has space
+    }
+
+    bool
+    transactionDbHasSpace(Config const& config) override
+    {
+        return true;  // In-memory database always has space
+    }
+
+    std::uint32_t
+    getKBUsedAll() override
+    {
+        std::uint32_t size = sizeof(*this);
+        size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData));
+        size +=
+            ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex));
+        size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx));
+        accountTxMap_.visit_all([&size](auto const& item) {
+            size += sizeof(AccountID) + sizeof(AccountTxData);
+            size += item.second.transactions.size() * sizeof(AccountTx);
+        });
+        return size / 1024;  // Convert to KB
+    }
+
+    std::uint32_t
+    getKBUsedLedger() override
+    {
+        std::uint32_t size =
+            ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData));
+        size +=
+            ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex));
+        return size / 1024;
+    }
+
+    std::uint32_t
+    getKBUsedTransaction() override
+    {
+        std::uint32_t size =
+            transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx));
+        accountTxMap_.visit_all([&size](auto const& item) {
+            size += sizeof(AccountID) + sizeof(AccountTxData);
+            size += item.second.transactions.size() * sizeof(AccountTx);
+        });
+        return size / 1024;
+    }
+
+    void
+    closeLedgerDB() override
+    {
+        // No-op for in-memory database
+    }
+
+    void
+    closeTransactionDB() override
+    {
+        // No-op for in-memory database
+    }
+
+    ~FlatmapDatabase()
+    {
+        // Concurrent maps need visit_all
+        accountTxMap_.visit_all(
+            [](auto& pair) { pair.second.transactions.clear(); });
+        accountTxMap_.clear();
+
+        transactionMap_.clear();
+
+        ledgers_.visit_all(
+            [](auto& pair) { pair.second.transactions.clear(); });
+        ledgers_.clear();
+
+        ledgerHashToSeq_.clear();
+    }
+
+    std::vector<std::shared_ptr<Transaction>>
+    getTxHistory(LedgerIndex startIndex) override
+    {
+        std::vector<std::shared_ptr<Transaction>> result;
+        transactionMap_.visit_all([&](auto const& item) {
+            if (item.second.second->getLgrSeq() >= startIndex)
+            {
+                result.push_back(item.second.first);
+            }
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a->getLedger() > b->getLedger();
+            });
+        if (result.size() > 20)
+        {
+            result.resize(20);
+        }
+        return result;
+    }
+    // Helper function to handle limits
+    template <typename Container>
+    void
+    applyLimit(Container& container, std::size_t limit, bool bUnlimited)
+    {
+        if (!bUnlimited && limit > 0 && container.size() > limit)
+        {
+            container.resize(limit);
+        }
+    }
+
+    AccountTxs
+    getOldestAccountTxs(AccountTxOptions const& options) override
+    {
+        AccountTxs result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.push_back(tx.second);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a.second->getLgrSeq() < b.second->getLgrSeq();
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    AccountTxs
+    getNewestAccountTxs(AccountTxOptions const& options) override
+    {
+        AccountTxs result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.push_back(tx.second);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a.second->getLgrSeq() > b.second->getLgrSeq();
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    MetaTxsList
+    getOldestAccountTxsB(AccountTxOptions const& options) override
+    {
+        MetaTxsList result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.emplace_back(
+                        tx.second.first->getSTransaction()
+                            ->getSerializer()
+                            .peekData(),
+                        tx.second.second->getAsObject()
+                            .getSerializer()
+                            .peekData(),
+                        tx.first.first);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return std::get<2>(a) < std::get<2>(b);
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    MetaTxsList
+    getNewestAccountTxsB(AccountTxOptions const& options) override
+    {
+        MetaTxsList result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.emplace_back(
+                        tx.second.first->getSTransaction()
+                            ->getSerializer()
+                            .peekData(),
+                        tx.second.second->getAsObject()
+                            .getSerializer()
+                            .peekData(),
+                        tx.first.first);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return std::get<2>(a) > std::get<2>(b);
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+    std::pair<AccountTxs, std::optional<AccountTxMarker>>
+    oldestAccountTxPage(AccountTxPageOptions const& options) override
+    {
+        AccountTxs result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::pair<std::pair<uint32_t, uint32_t>, AccountTx>>
+                txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(tx);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) {
+                return a.first < b.first;
+            });
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return tx.first.first == options.marker->ledgerSeq &&
+                        tx.first.second == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                result.push_back(it->second);
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{it->first.first, it->first.second};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<AccountTxs, std::optional<AccountTxMarker>>
+    newestAccountTxPage(AccountTxPageOptions const& options) override
+    {
+        AccountTxs result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::pair<std::pair<uint32_t, uint32_t>, AccountTx>>
+                txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(tx);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) {
+                return a.first > b.first;
+            });
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return tx.first.first == options.marker->ledgerSeq &&
+                        tx.first.second == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                result.push_back(it->second);
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{it->first.first, it->first.second};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<MetaTxsList, std::optional<AccountTxMarker>>
+    oldestAccountTxPageB(AccountTxPageOptions const& options) override
+    {
+        MetaTxsList result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::tuple<uint32_t, uint32_t, AccountTx>> txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(
+                        tx.first.first, tx.first.second, tx.second);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end());
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return std::get<0>(tx) == options.marker->ledgerSeq &&
+                        std::get<1>(tx) == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                const auto& [_, __, tx] = *it;
+                result.emplace_back(
+                    tx.first->getSTransaction()->getSerializer().peekData(),
+                    tx.second->getAsObject().getSerializer().peekData(),
+                    std::get<0>(*it));
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<MetaTxsList, std::optional<AccountTxMarker>>
+    newestAccountTxPageB(AccountTxPageOptions const& options) override
+    {
+        MetaTxsList result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::tuple<uint32_t, uint32_t, AccountTx>> txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(
+                        tx.first.first, tx.first.second, tx.second);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), std::greater<>());
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return std::get<0>(tx) == options.marker->ledgerSeq &&
+                        std::get<1>(tx) == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                const auto& [_, __, tx] = *it;
+                result.emplace_back(
+                    tx.first->getSTransaction()->getSerializer().peekData(),
+                    tx.second->getAsObject().getSerializer().peekData(),
+                    std::get<0>(*it));
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)};
+            }
+        });
+
+        return {result, marker};
+    }
+};
+
+// Factory function
+std::unique_ptr<SQLiteDatabase>
+getFlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue)
+{
+    return std::make_unique<FlatmapDatabase>(app, config, jobQueue);
+}
+
+}  // namespace ripple
+#endif  // RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED

src/ripple/app/rdb/backend/detail/impl/Node.cpp

@@ -1132,7 +1132,7 @@ accountTxPage(
     {
         sql = boost::str(
             boost::format(
-                prefix + (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u'
+                prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u
              ORDER BY AccountTransactions.LedgerSeq %s,
              AccountTransactions.TxnSeq %s
              LIMIT %u;)")) %
@@ -1155,12 +1155,14 @@ accountTxPage(
             FROM AccountTransactions, Transactions WHERE
             (AccountTransactions.TransID = Transactions.TransID AND
             AccountTransactions.Account = '%s' AND
-            AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u')
-            OR
+            AccountTransactions.LedgerSeq BETWEEN %u AND %u)
+            UNION
+            SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta
+            FROM AccountTransactions, Transactions WHERE
             (AccountTransactions.TransID = Transactions.TransID AND
             AccountTransactions.Account = '%s' AND
-            AccountTransactions.LedgerSeq = '%u' AND
-            AccountTransactions.TxnSeq %s '%u')
+            AccountTransactions.LedgerSeq = %u AND
+            AccountTransactions.TxnSeq %s %u)
             ORDER BY AccountTransactions.LedgerSeq %s,
             AccountTransactions.TxnSeq %s
             LIMIT %u;

src/ripple/app/rdb/backend/impl/SQLiteDatabase.cpp

@@ -601,11 +601,44 @@ SQLiteDatabaseImp::deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq)
 void
 SQLiteDatabaseImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq)
 {
+    // Get a reference to the pinned ledgers set for quick lookups
+    RangeSet<std::uint32_t> pinnedLedgers;
+    {
+        auto& ledgerMaster = app_.getLedgerMaster();
+        // Use public API to get the pinned ledgers
+        pinnedLedgers = ledgerMaster.getPinnedLedgersRangeSet();
+    }
+
     if (existsLedger())
     {
         auto db = checkoutLedger();
-        detail::deleteBeforeLedgerSeq(
-            *db, detail::TableType::Ledgers, ledgerSeq);
+
+        // Check if any ledgers in the range to be deleted are pinned
+        bool hasPinnedLedgers = false;
+        for (LedgerIndex seq = 1; seq < ledgerSeq && !hasPinnedLedgers; ++seq)
+        {
+            if (boost::icl::contains(pinnedLedgers, seq))
+                hasPinnedLedgers = true;
+        }
+
+        if (!hasPinnedLedgers)
+        {
+            // No pinned ledgers in the range, proceed with normal delete
+            detail::deleteBeforeLedgerSeq(
+                *db, detail::TableType::Ledgers, ledgerSeq);
+        }
+        else
+        {
+            // Delete ledgers individually, skipping pinned ones
+            for (LedgerIndex seq = 1; seq < ledgerSeq; ++seq)
+            {
+                if (!boost::icl::contains(pinnedLedgers, seq))
+                {
+                    detail::deleteByLedgerSeq(
+                        *db, detail::TableType::Ledgers, seq);
+                }
+            }
+        }
         return;
     }
 
@@ -614,8 +647,39 @@ SQLiteDatabaseImp::deleteBeforeLedgerSeq(LedgerIndex ledgerSeq)
         iterateLedgerBack(
             seqToShardIndex(ledgerSeq),
             [&](soci::session& session, std::uint32_t shardIndex) {
-                detail::deleteBeforeLedgerSeq(
-                    session, detail::TableType::Ledgers, ledgerSeq);
+                LedgerIndex firstSeq = firstLedgerSeq(shardIndex);
+                LedgerIndex lastSeq =
+                    std::min(lastLedgerSeq(shardIndex), ledgerSeq - 1);
+
+                // Check if any ledgers in this shard's range are pinned
+                bool hasPinnedLedgers = false;
+                for (LedgerIndex seq = firstSeq;
+                     seq <= lastSeq && !hasPinnedLedgers;
+                     ++seq)
+                {
+                    if (boost::icl::contains(pinnedLedgers, seq))
+                        hasPinnedLedgers = true;
+                }
+
+                if (!hasPinnedLedgers)
+                {
+                    // No pinned ledgers in this shard range, proceed with
+                    // normal delete
+                    detail::deleteBeforeLedgerSeq(
+                        session, detail::TableType::Ledgers, ledgerSeq);
+                }
+                else
+                {
+                    // Delete ledgers individually, skipping pinned ones
+                    for (LedgerIndex seq = firstSeq; seq <= lastSeq; ++seq)
+                    {
+                        if (!boost::icl::contains(pinnedLedgers, seq))
+                        {
+                            detail::deleteByLedgerSeq(
+                                session, detail::TableType::Ledgers, seq);
+                        }
+                    }
+                }
                 return true;
             });
     }
@@ -627,11 +691,43 @@ SQLiteDatabaseImp::deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq)
     if (!useTxTables_)
         return;
 
+    // Get a reference to the pinned ledgers set for quick lookups
+    RangeSet<std::uint32_t> pinnedLedgers;
+    {
+        auto& ledgerMaster = app_.getLedgerMaster();
+        pinnedLedgers = ledgerMaster.getPinnedLedgersRangeSet();
+    }
+
     if (existsTransaction())
     {
         auto db = checkoutTransaction();
-        detail::deleteBeforeLedgerSeq(
-            *db, detail::TableType::Transactions, ledgerSeq);
+
+        // Check if any ledgers in the range to be deleted are pinned
+        bool hasPinnedLedgers = false;
+        for (LedgerIndex seq = 1; seq < ledgerSeq && !hasPinnedLedgers; ++seq)
+        {
+            if (boost::icl::contains(pinnedLedgers, seq))
+                hasPinnedLedgers = true;
+        }
+
+        if (!hasPinnedLedgers)
+        {
+            // No pinned ledgers in the range, proceed with normal delete
+            detail::deleteBeforeLedgerSeq(
+                *db, detail::TableType::Transactions, ledgerSeq);
+        }
+        else
+        {
+            // Delete transaction data individually, skipping pinned ledgers
+            for (LedgerIndex seq = 1; seq < ledgerSeq; ++seq)
+            {
+                if (!boost::icl::contains(pinnedLedgers, seq))
+                {
+                    detail::deleteByLedgerSeq(
+                        *db, detail::TableType::Transactions, seq);
+                }
+            }
+        }
         return;
     }
 
@@ -640,8 +736,40 @@ SQLiteDatabaseImp::deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq)
         iterateTransactionBack(
             seqToShardIndex(ledgerSeq),
             [&](soci::session& session, std::uint32_t shardIndex) {
-                detail::deleteBeforeLedgerSeq(
-                    session, detail::TableType::Transactions, ledgerSeq);
+                LedgerIndex firstSeq = firstLedgerSeq(shardIndex);
+                LedgerIndex lastSeq =
+                    std::min(lastLedgerSeq(shardIndex), ledgerSeq - 1);
+
+                // Check if any ledgers in this shard's range are pinned
+                bool hasPinnedLedgers = false;
+                for (LedgerIndex seq = firstSeq;
+                     seq <= lastSeq && !hasPinnedLedgers;
+                     ++seq)
+                {
+                    if (boost::icl::contains(pinnedLedgers, seq))
+                        hasPinnedLedgers = true;
+                }
+
+                if (!hasPinnedLedgers)
+                {
+                    // No pinned ledgers in this shard range, proceed with
+                    // normal delete
+                    detail::deleteBeforeLedgerSeq(
+                        session, detail::TableType::Transactions, ledgerSeq);
+                }
+                else
+                {
+                    // Delete transaction data individually, skipping pinned
+                    // ledgers
+                    for (LedgerIndex seq = firstSeq; seq <= lastSeq; ++seq)
+                    {
+                        if (!boost::icl::contains(pinnedLedgers, seq))
+                        {
+                            detail::deleteByLedgerSeq(
+                                session, detail::TableType::Transactions, seq);
+                        }
+                    }
+                }
                 return true;
             });
     }
@@ -654,11 +782,44 @@ SQLiteDatabaseImp::deleteAccountTransactionsBeforeLedgerSeq(
     if (!useTxTables_)
         return;
 
+    // Get a reference to the pinned ledgers set for quick lookups
+    RangeSet<std::uint32_t> pinnedLedgers;
+    {
+        auto& ledgerMaster = app_.getLedgerMaster();
+        pinnedLedgers = ledgerMaster.getPinnedLedgersRangeSet();
+    }
+
     if (existsTransaction())
     {
         auto db = checkoutTransaction();
-        detail::deleteBeforeLedgerSeq(
-            *db, detail::TableType::AccountTransactions, ledgerSeq);
+
+        // Check if any ledgers in the range to be deleted are pinned
+        bool hasPinnedLedgers = false;
+        for (LedgerIndex seq = 1; seq < ledgerSeq && !hasPinnedLedgers; ++seq)
+        {
+            if (boost::icl::contains(pinnedLedgers, seq))
+                hasPinnedLedgers = true;
+        }
+
+        if (!hasPinnedLedgers)
+        {
+            // No pinned ledgers in the range, proceed with normal delete
+            detail::deleteBeforeLedgerSeq(
+                *db, detail::TableType::AccountTransactions, ledgerSeq);
+        }
+        else
+        {
+            // Delete account transaction data individually, skipping pinned
+            // ledgers
+            for (LedgerIndex seq = 1; seq < ledgerSeq; ++seq)
+            {
+                if (!boost::icl::contains(pinnedLedgers, seq))
+                {
+                    detail::deleteByLedgerSeq(
+                        *db, detail::TableType::AccountTransactions, seq);
+                }
+            }
+        }
         return;
     }
 
@@ -667,8 +828,44 @@ SQLiteDatabaseImp::deleteAccountTransactionsBeforeLedgerSeq(
         iterateTransactionBack(
             seqToShardIndex(ledgerSeq),
             [&](soci::session& session, std::uint32_t shardIndex) {
-                detail::deleteBeforeLedgerSeq(
-                    session, detail::TableType::AccountTransactions, ledgerSeq);
+                LedgerIndex firstSeq = firstLedgerSeq(shardIndex);
+                LedgerIndex lastSeq =
+                    std::min(lastLedgerSeq(shardIndex), ledgerSeq - 1);
+
+                // Check if any ledgers in this shard's range are pinned
+                bool hasPinnedLedgers = false;
+                for (LedgerIndex seq = firstSeq;
+                     seq <= lastSeq && !hasPinnedLedgers;
+                     ++seq)
+                {
+                    if (boost::icl::contains(pinnedLedgers, seq))
+                        hasPinnedLedgers = true;
+                }
+
+                if (!hasPinnedLedgers)
+                {
+                    // No pinned ledgers in this shard range, proceed with
+                    // normal delete
+                    detail::deleteBeforeLedgerSeq(
+                        session,
+                        detail::TableType::AccountTransactions,
+                        ledgerSeq);
+                }
+                else
+                {
+                    // Delete account transaction data individually, skipping
+                    // pinned ledgers
+                    for (LedgerIndex seq = firstSeq; seq <= lastSeq; ++seq)
+                    {
+                        if (!boost::icl::contains(pinnedLedgers, seq))
+                        {
+                            detail::deleteByLedgerSeq(
+                                session,
+                                detail::TableType::AccountTransactions,
+                                seq);
+                        }
+                    }
+                }
                 return true;
             });
     }

src/ripple/app/rdb/impl/RelationalDatabase.cpp

@@ -19,6 +19,8 @@
 
 #include <ripple/app/main/Application.h>
 #include <ripple/app/rdb/RelationalDatabase.h>
+#include <ripple/app/rdb/backend/FlatmapDatabase.h>
+#include <ripple/app/rdb/backend/RWDBDatabase.h>
 #include <ripple/core/ConfigSections.h>
 #include <ripple/nodestore/DatabaseShard.h>
 
@@ -38,6 +40,8 @@ RelationalDatabase::init(
 {
     bool use_sqlite = false;
     bool use_postgres = false;
+    bool use_rwdb = false;
+    bool use_flatmap = false;
 
     if (config.reporting())
     {
@@ -52,6 +56,14 @@ RelationalDatabase::init(
             {
                 use_sqlite = true;
             }
+            else if (boost::iequals(get(rdb_section, "backend"), "rwdb"))
+            {
+                use_rwdb = true;
+            }
+            else if (boost::iequals(get(rdb_section, "backend"), "flatmap"))
+            {
+                use_flatmap = true;
+            }
             else
             {
                 Throw<std::runtime_error>(
@@ -73,6 +85,14 @@ RelationalDatabase::init(
     {
         return getPostgresDatabase(app, config, jobQueue);
     }
+    else if (use_rwdb)
+    {
+        return getRWDBDatabase(app, config, jobQueue);
+    }
+    else if (use_flatmap)
+    {
+        return getFlatmapDatabase(app, config, jobQueue);
+    }
 
     return std::unique_ptr<RelationalDatabase>();
 }

src/ripple/app/rdb/impl/UnitaryShard.cpp

@@ -103,22 +103,24 @@ updateLedgerDBs(
 
             for (auto const& item : ledger->txs)
             {
-                if (stop)
+                if (stop.load(std::memory_order_relaxed))
                     return false;
 
-                auto const txID{item.first->getTransactionID()};
-                auto const sTxID{to_string(txID)};
-                auto const txMeta{std::make_shared<TxMeta>(
-                    txID, ledger->seq(), *item.second)};
+                TxMeta const txMeta{
+                    item.first->getTransactionID(),
+                    ledger->seq(),
+                    *item.second};
+
+                auto const sTxID = to_string(txMeta.getTxID());
 
                 session << "DELETE FROM AccountTransactions "
                            "WHERE TransID = :txID;",
                     soci::use(sTxID);
 
-                auto const& accounts = txMeta->getAffectedAccounts();
+                auto const& accounts = txMeta.getAffectedAccounts();
                 if (!accounts.empty())
                 {
-                    auto const sTxnSeq{std::to_string(txMeta->getIndex())};
+                    auto const sTxnSeq{std::to_string(txMeta.getIndex())};
                     auto const s{boost::str(
                         boost::format("('%s','%s',%s,%s)") % sTxID % "%s" %
                         sSeq % sTxnSeq)};

src/ripple/app/tx/impl/Change.cpp

@@ -458,6 +458,13 @@ Change::activateXahauGenesis()
     bool const isTest =
         (ctx_.tx.getFlags() & tfTestSuite) && ctx_.app.config().standalone();
 
+    // RH NOTE: we'll only configure xahau governance structure on networks that
+    // begin with 2133... so production xahau: 21337 and its testnet 21338
+    // with 21330-21336 and 21339 also valid and reserved for dev nets etc.
+    // all other Network IDs will be conventionally configured.
+    if ((ctx_.app.config().NETWORK_ID / 10) != 2133 && !isTest)
+        return;
+
     auto [ng_entries, l1_entries, l2_entries, gov_params] =
         normalizeXahauGenesis(
             isTest ? TestNonGovernanceDistribution : NonGovernanceDistribution,
@@ -580,7 +587,8 @@ Change::activateXahauGenesis()
                 wasmBytes,  // wasm to verify
                 loggerStream,
                 "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
-                ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0);
+                (ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0) +
+                    (ctx_.view().rules().enabled(fix20250131) ? 2 : 0));
 
             if (!result)
             {

src/ripple/app/tx/impl/Import.cpp

@@ -889,6 +889,45 @@ Import::preclaim(PreclaimContext const& ctx)
     }
 
     auto const& sle = ctx.view.read(keylet::account(ctx.tx[sfAccount]));
+
+    auto const tt = stpTrans->getTxnType();
+    if ((tt == ttSIGNER_LIST_SET || tt == ttREGULAR_KEY_SET) &&
+        ctx.view.rules().enabled(fixReduceImport) && sle)
+    {
+        // blackhole check
+        do
+        {
+            // if master key is not set then it is not blackholed
+            if (!(sle->getFlags() & lsfDisableMaster))
+                break;
+
+            // if a regular key is set then it must be acc 0, 1, or 2 otherwise
+            // not blackholed
+            if (sle->isFieldPresent(sfRegularKey))
+            {
+                AccountID rk = sle->getAccountID(sfRegularKey);
+                static const AccountID ACCOUNT_ZERO(0);
+                static const AccountID ACCOUNT_ONE(1);
+                static const AccountID ACCOUNT_TWO(2);
+
+                if (rk != ACCOUNT_ZERO && rk != ACCOUNT_ONE &&
+                    rk != ACCOUNT_TWO)
+                    break;
+            }
+
+            // if a signer list is set then it's not blackholed
+            auto const signerListKeylet = keylet::signers(ctx.tx[sfAccount]);
+            if (ctx.view.exists(signerListKeylet))
+                break;
+
+            // execution to here means it's blackholed
+            JLOG(ctx.j.warn())
+                << "Import: during preclaim target account is blackholed "
+                << ctx.tx[sfAccount] << ", bailing.";
+            return tefIMPORT_BLACKHOLED;
+        } while (0);
+    }
+
     if (sle && sle->isFieldPresent(sfImportSequence))
     {
         uint32_t sleImportSequence = sle->getFieldU32(sfImportSequence);

src/ripple/app/tx/impl/InvariantCheck.h

@@ -319,6 +319,17 @@ public:
         beast::Journal const&);
 };
 
+/**
+ * @brief Invariant: Validates several invariants for NFToken pages.
+ *
+ * The following checks are made:
+ *  - The page is correctly associated with the owner.
+ *  - The page is correctly ordered between the next and previous links.
+ *  - The page contains at least one and no more than 32 NFTokens.
+ *  - The NFTokens on this page do not belong on a lower or higher page.
+ *  - The NFTokens are correctly sorted on the page.
+ *  - Each URI, if present, is not empty.
+ */
 class ValidNFTokenPage
 {
     bool badEntry_ = false;
@@ -343,6 +354,19 @@ public:
         beast::Journal const&);
 };
 
+/**
+ * @brief Invariant: Validates counts of NFTokens after all transaction types.
+ *
+ * The following checks are made:
+ *  - The number of minted or burned NFTokens can only be changed by
+ *    NFTokenMint or NFTokenBurn transactions.
+ *  - A successful NFTokenMint must increase the number of NFTokens.
+ *  - A failed NFTokenMint must not change the number of minted NFTokens.
+ *  - An NFTokenMint transaction cannot change the number of burned NFTokens.
+ *  - A successful NFTokenBurn must increase the number of burned NFTokens.
+ *  - A failed NFTokenBurn must not change the number of burned NFTokens.
+ *  - An NFTokenBurn transaction cannot change the number of minted NFTokens.
+ */
 class NFTokenCountTracking
 {
     std::uint32_t beforeMintedTotal = 0;

src/ripple/app/tx/impl/Remit.cpp

@@ -72,6 +72,16 @@ Remit::preflight(PreflightContext const& ctx)
         return temREDUNDANT;
     }
 
+    if (ctx.rules.enabled(fix20250131))
+    {
+        if (!dstID || dstID == noAccount())
+        {
+            JLOG(ctx.j.warn())
+                << "Malformed transaction: Remit to invalid account.";
+            return temMALFORMED;
+        }
+    }
+
     if (ctx.tx.isFieldPresent(sfInform))
     {
         AccountID const infID = ctx.tx.getAccountID(sfInform);

src/ripple/app/tx/impl/SetHook.cpp

@@ -479,7 +479,8 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj)
                     hook,  // wasm to verify
                     logger,
                     hsacc,
-                    ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0);
+                    (ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0) +
+                        (ctx.rules.enabled(fix20250131) ? 2 : 0));
 
                 if (ctx.j.trace())
                 {

src/ripple/app/tx/impl/Transactor.cpp

@@ -1079,6 +1079,24 @@ Transactor::checkMultiSign(PreclaimContext const& ctx)
 
 //------------------------------------------------------------------------------
 
+// increment the touch counter on an account
+static void
+touchAccount(ApplyView& view, AccountID const& id)
+{
+    if (!view.rules().enabled(featureTouch))
+        return;
+
+    std::shared_ptr<SLE> sle = view.peek(keylet::account(id));
+    if (!sle)
+        return;
+
+    uint64_t tc =
+        sle->isFieldPresent(sfTouchCount) ? sle->getFieldU64(sfTouchCount) : 0;
+
+    sle->setFieldU64(sfTouchCount, tc + 1);
+    view.update(sle);
+}
+
 static void
 removeUnfundedOffers(
     ApplyView& view,
@@ -1270,10 +1288,18 @@ Transactor::executeHookChain(
                 if (results.back().exitType == hook_api::ExitType::WASM_ERROR)
                 {
                     JLOG(j_.warn()) << "HookError[" << account << "-"
-                                    << ctx_.tx.getAccountID(sfAccount) << "]: "
+                                    << ctx_.tx.getAccountID(sfAccount)
                                     << "]: Execution failure (graceful) "
                                     << "HookHash: " << hookHash;
                 }
+                if (results.back().exitType == hook_api::ExitType::UNSET)
+                {
+                    JLOG(j_.warn())
+                        << "HookError[" << account << "-"
+                        << ctx_.tx.getAccountID(sfAccount)
+                        << "]: Execution failure (no exit type specified) "
+                        << "HookHash: " << hookHash;
+                }
                 return tecHOOK_REJECTED;
             }
 
@@ -1298,7 +1324,7 @@ Transactor::executeHookChain(
         {
             JLOG(j_.warn())
                 << "HookError[" << account << "-"
-                << ctx_.tx.getAccountID(sfAccount) << "]: "
+                << ctx_.tx.getAccountID(sfAccount)
                 << "]: Execution failure (exceptional) "
                 << "Exception: " << e.what() << " HookHash: " << hookHash;
 
@@ -1426,13 +1452,13 @@ Transactor::doHookCallback(
                 finalizeHookResult(callbackResult, ctx_, success);
 
             JLOG(j_.trace()) << "HookInfo[" << callbackAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
-                             << "Callback finalizeHookResult = " << result;
+                             << ctx_.tx.getAccountID(sfAccount)
+                             << "]: Callback finalizeHookResult = " << result;
         }
         catch (std::exception& e)
         {
             JLOG(j_.fatal()) << "HookError[" << callbackAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
+                             << ctx_.tx.getAccountID(sfAccount)
                              << "]: Callback failure " << e.what();
         }
     }
@@ -1511,6 +1537,8 @@ Transactor::doTSH(
         if ((!canRollback && strong) || (canRollback && !strong))
             continue;
 
+        touchAccount(view, tshAccountID);
+
         auto klTshHook = keylet::hook(tshAccountID);
 
         auto tshHook = view.read(klTshHook);
@@ -1678,13 +1706,13 @@ Transactor::doAgainAsWeak(
             results.push_back(aawResult);
 
             JLOG(j_.trace()) << "HookInfo[" << hookAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
-                             << " aaw Hook ExitCode = " << aawResult.exitCode;
+                             << ctx_.tx.getAccountID(sfAccount)
+                             << "]: aaw Hook ExitCode = " << aawResult.exitCode;
         }
         catch (std::exception& e)
         {
             JLOG(j_.fatal()) << "HookError[" << hookAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
+                             << ctx_.tx.getAccountID(sfAccount)
                              << "]: aaw failure " << e.what();
         }
     }

src/ripple/basics/IOUAmount.h

@@ -186,7 +186,14 @@ mulRatio(
     std::uint32_t den,
     bool roundUp);
 
-extern LocalValue<bool> stNumberSwitchover;
+// Since many uses of the number class do not have access to a ledger,
+// getSTNumberSwitchover needs to be globally accessible.
+
+bool
+getSTNumberSwitchover();
+
+void
+setSTNumberSwitchover(bool v);
 
 /** RAII class to set and restore the Number switchover.
  */
@@ -198,16 +205,16 @@ class NumberSO
 public:
     ~NumberSO()
     {
-        *stNumberSwitchover = saved_;
+        setSTNumberSwitchover(saved_);
     }
 
     NumberSO(NumberSO const&) = delete;
     NumberSO&
     operator=(NumberSO const&) = delete;
 
-    explicit NumberSO(bool v) : saved_(*stNumberSwitchover)
+    explicit NumberSO(bool v) : saved_(getSTNumberSwitchover())
     {
-        *stNumberSwitchover = v;
+        setSTNumberSwitchover(v);
     }
 };
 

src/ripple/basics/SlabAllocator.h

@@ -0,0 +1,432 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright 2022, Nikolaos D. Bougalis <nikb@bougalis.net>
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED
+#define RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED
+
+#include <ripple/beast/type_name.h>
+
+#include <boost/align.hpp>
+#include <boost/container/static_vector.hpp>
+#include <boost/predef.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+#include <mutex>
+
+#if BOOST_OS_LINUX
+#include <sys/mman.h>
+#endif
+
+namespace ripple {
+
+template <typename Type>
+class SlabAllocator
+{
+    static_assert(
+        sizeof(Type) >= sizeof(std::uint8_t*),
+        "SlabAllocator: the requested object must be larger than a pointer.");
+
+    static_assert(alignof(Type) == 8 || alignof(Type) == 4);
+
+    /** A block of memory that is owned by a slab allocator */
+    struct SlabBlock
+    {
+        // A mutex to protect the freelist for this block:
+        std::mutex m_;
+
+        // A linked list of appropriately sized free buffers:
+        std::uint8_t* l_ = nullptr;
+
+        // The next memory block
+        SlabBlock* next_;
+
+        // The underlying memory block:
+        std::uint8_t const* const p_ = nullptr;
+
+        // The extent of the underlying memory block:
+        std::size_t const size_;
+
+        SlabBlock(
+            SlabBlock* next,
+            std::uint8_t* data,
+            std::size_t size,
+            std::size_t item)
+            : next_(next), p_(data), size_(size)
+        {
+            // We don't need to grab the mutex here, since we're the only
+            // ones with access at this moment.
+
+            while (data + item <= p_ + size_)
+            {
+                // Use memcpy to avoid unaligned UB
+                // (will optimize to equivalent code)
+                std::memcpy(data, &l_, sizeof(std::uint8_t*));
+                l_ = data;
+                data += item;
+            }
+        }
+
+        ~SlabBlock()
+        {
+            // Calling this destructor will release the allocated memory but
+            // will not properly destroy any objects that are constructed in
+            // the block itself.
+        }
+
+        SlabBlock(SlabBlock const& other) = delete;
+        SlabBlock&
+        operator=(SlabBlock const& other) = delete;
+
+        SlabBlock(SlabBlock&& other) = delete;
+        SlabBlock&
+        operator=(SlabBlock&& other) = delete;
+
+        /** Determines whether the given pointer belongs to this allocator */
+        bool
+        own(std::uint8_t const* p) const noexcept
+        {
+            return (p >= p_) && (p < p_ + size_);
+        }
+
+        std::uint8_t*
+        allocate() noexcept
+        {
+            std::uint8_t* ret;
+
+            {
+                std::lock_guard l(m_);
+
+                ret = l_;
+
+                if (ret)
+                {
+                    // Use memcpy to avoid unaligned UB
+                    // (will optimize to equivalent code)
+                    std::memcpy(&l_, ret, sizeof(std::uint8_t*));
+                }
+            }
+
+            return ret;
+        }
+
+        /** Return an item to this allocator's freelist.
+
+            @param ptr The pointer to the chunk of memory being deallocated.
+
+            @note This is a dangerous, private interface; the item being
+                  returned should belong to this allocator. Debug builds
+                  will check and assert if this is not the case. Release
+                  builds will not.
+         */
+        void
+        deallocate(std::uint8_t* ptr) noexcept
+        {
+            assert(own(ptr));
+
+            std::lock_guard l(m_);
+
+            // Use memcpy to avoid unaligned UB
+            // (will optimize to equivalent code)
+            std::memcpy(ptr, &l_, sizeof(std::uint8_t*));
+            l_ = ptr;
+        }
+    };
+
+private:
+    // A linked list of slabs
+    std::atomic<SlabBlock*> slabs_ = nullptr;
+
+    // The alignment requirements of the item we're allocating:
+    std::size_t const itemAlignment_;
+
+    // The size of an item, including the extra bytes requested and
+    // any padding needed for alignment purposes:
+    std::size_t const itemSize_;
+
+    // The size of each individual slab:
+    std::size_t const slabSize_;
+
+public:
+    /** Constructs a slab allocator able to allocate objects of a fixed size
+
+        @param count the number of items the slab allocator can allocate; note
+                     that a count of 0 is valid and means that the allocator
+                     is, effectively, disabled. This can be very useful in some
+                     contexts (e.g. when mimimal memory usage is needed) and
+                     allows for graceful failure.
+     */
+    constexpr explicit SlabAllocator(
+        std::size_t extra,
+        std::size_t alloc = 0,
+        std::size_t align = 0)
+        : itemAlignment_(align ? align : alignof(Type))
+        , itemSize_(
+              boost::alignment::align_up(sizeof(Type) + extra, itemAlignment_))
+        , slabSize_(alloc)
+    {
+        assert((itemAlignment_ & (itemAlignment_ - 1)) == 0);
+    }
+
+    SlabAllocator(SlabAllocator const& other) = delete;
+    SlabAllocator&
+    operator=(SlabAllocator const& other) = delete;
+
+    SlabAllocator(SlabAllocator&& other) = delete;
+    SlabAllocator&
+    operator=(SlabAllocator&& other) = delete;
+
+    ~SlabAllocator()
+    {
+        // FIXME: We can't destroy the memory blocks we've allocated, because
+        //        we can't be sure that they are not being used. Cleaning the
+        //        shutdown process up could make this possible.
+    }
+
+    /** Returns the size of the memory block this allocator returns. */
+    constexpr std::size_t
+    size() const noexcept
+    {
+        return itemSize_;
+    }
+
+    /** Returns a suitably aligned pointer, if one is available.
+
+        @return a pointer to a block of memory from the allocator, or
+                nullptr if the allocator can't satisfy this request.
+     */
+    std::uint8_t*
+    allocate() noexcept
+    {
+        auto slab = slabs_.load();
+
+        while (slab != nullptr)
+        {
+            if (auto ret = slab->allocate())
+                return ret;
+
+            slab = slab->next_;
+        }
+
+        // No slab can satisfy our request, so we attempt to allocate a new
+        // one here:
+        std::size_t size = slabSize_;
+
+        // We want to allocate the memory at a 2 MiB boundary, to make it
+        // possible to use hugepage mappings on Linux:
+        auto buf =
+            boost::alignment::aligned_alloc(megabytes(std::size_t(2)), size);
+
+        // clang-format off
+        if (!buf) [[unlikely]]
+            return nullptr;
+            // clang-format on
+
+#if BOOST_OS_LINUX
+        // When allocating large blocks, attempt to leverage Linux's
+        // transparent hugepage support. It is unclear and difficult
+        // to accurately determine if doing this impacts performance
+        // enough to justify using platform-specific tricks.
+        if (size >= megabytes(std::size_t(4)))
+            madvise(buf, size, MADV_HUGEPAGE);
+#endif
+
+        // We need to carve out a bit of memory for the slab header
+        // and then align the rest appropriately:
+        auto slabData = reinterpret_cast<void*>(
+            reinterpret_cast<std::uint8_t*>(buf) + sizeof(SlabBlock));
+        auto slabSize = size - sizeof(SlabBlock);
+
+        // This operation is essentially guaranteed not to fail but
+        // let's be careful anyways.
+        if (!boost::alignment::align(
+                itemAlignment_, itemSize_, slabData, slabSize))
+        {
+            boost::alignment::aligned_free(buf);
+            return nullptr;
+        }
+
+        slab = new (buf) SlabBlock(
+            slabs_.load(),
+            reinterpret_cast<std::uint8_t*>(slabData),
+            slabSize,
+            itemSize_);
+
+        // Link the new slab
+        while (!slabs_.compare_exchange_weak(
+            slab->next_,
+            slab,
+            std::memory_order_release,
+            std::memory_order_relaxed))
+        {
+            ;  // Nothing to do
+        }
+
+        return slab->allocate();
+    }
+
+    /** Returns the memory block to the allocator.
+
+        @param ptr A pointer to a memory block.
+        @param size If non-zero, a hint as to the size of the block.
+        @return true if this memory block belonged to the allocator and has
+                     been released; false otherwise.
+     */
+    bool
+    deallocate(std::uint8_t* ptr) noexcept
+    {
+        assert(ptr);
+
+        for (auto slab = slabs_.load(); slab != nullptr; slab = slab->next_)
+        {
+            if (slab->own(ptr))
+            {
+                slab->deallocate(ptr);
+                return true;
+            }
+        }
+
+        return false;
+    }
+};
+
+/** A collection of slab allocators of various sizes for a given type. */
+template <typename Type>
+class SlabAllocatorSet
+{
+private:
+    // The list of allocators that belong to this set
+    boost::container::static_vector<SlabAllocator<Type>, 64> allocators_;
+
+    std::size_t maxSize_ = 0;
+
+public:
+    class SlabConfig
+    {
+        friend class SlabAllocatorSet;
+
+    private:
+        std::size_t extra;
+        std::size_t alloc;
+        std::size_t align;
+
+    public:
+        constexpr SlabConfig(
+            std::size_t extra_,
+            std::size_t alloc_ = 0,
+            std::size_t align_ = alignof(Type))
+            : extra(extra_), alloc(alloc_), align(align_)
+        {
+        }
+    };
+
+    constexpr SlabAllocatorSet(std::vector<SlabConfig> cfg)
+    {
+        // Ensure that the specified allocators are sorted from smallest to
+        // largest by size:
+        std::sort(
+            std::begin(cfg),
+            std::end(cfg),
+            [](SlabConfig const& a, SlabConfig const& b) {
+                return a.extra < b.extra;
+            });
+
+        // We should never have two slabs of the same size
+        if (std::adjacent_find(
+                std::begin(cfg),
+                std::end(cfg),
+                [](SlabConfig const& a, SlabConfig const& b) {
+                    return a.extra == b.extra;
+                }) != cfg.end())
+        {
+            throw std::runtime_error(
+                "SlabAllocatorSet<" + beast::type_name<Type>() +
+                ">: duplicate slab size");
+        }
+
+        for (auto const& c : cfg)
+        {
+            auto& a = allocators_.emplace_back(c.extra, c.alloc, c.align);
+
+            if (a.size() > maxSize_)
+                maxSize_ = a.size();
+        }
+    }
+
+    SlabAllocatorSet(SlabAllocatorSet const& other) = delete;
+    SlabAllocatorSet&
+    operator=(SlabAllocatorSet const& other) = delete;
+
+    SlabAllocatorSet(SlabAllocatorSet&& other) = delete;
+    SlabAllocatorSet&
+    operator=(SlabAllocatorSet&& other) = delete;
+
+    ~SlabAllocatorSet()
+    {
+    }
+
+    /** Returns a suitably aligned pointer, if one is available.
+
+        @param extra The number of extra bytes, above and beyond the size of
+                     the object, that should be returned by the allocator.
+
+        @return a pointer to a block of memory, or nullptr if the allocator
+                can't satisfy this request.
+     */
+    std::uint8_t*
+    allocate(std::size_t extra) noexcept
+    {
+        if (auto const size = sizeof(Type) + extra; size <= maxSize_)
+        {
+            for (auto& a : allocators_)
+            {
+                if (a.size() >= size)
+                    return a.allocate();
+            }
+        }
+
+        return nullptr;
+    }
+
+    /** Returns the memory block to the allocator.
+
+        @param ptr A pointer to a memory block.
+
+        @return true if this memory block belonged to one of the allocators
+                     in this set and has been released; false otherwise.
+     */
+    bool
+    deallocate(std::uint8_t* ptr) noexcept
+    {
+        for (auto& a : allocators_)
+        {
+            if (a.deallocate(ptr))
+                return true;
+        }
+
+        return false;
+    }
+};
+
+}  // namespace ripple
+
+#endif  // RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED

src/ripple/basics/StringUtilities.h

@@ -25,7 +25,9 @@
 
 #include <boost/format.hpp>
 #include <boost/utility/string_view.hpp>
+
 #include <array>
+#include <cstdint>
 #include <optional>
 #include <sstream>
 #include <string>

src/ripple/basics/base64.h

@@ -57,6 +57,7 @@
 #ifndef RIPPLE_BASICS_BASE64_H_INCLUDED
 #define RIPPLE_BASICS_BASE64_H_INCLUDED
 
+#include <cstdint>
 #include <string>
 
 namespace ripple {

src/ripple/basics/base_uint.h

@@ -129,6 +129,18 @@ public:
         return reinterpret_cast<const_pointer>(data_.data());
     }
 
+    char const*
+    cdata() const
+    {
+        return reinterpret_cast<char const*>(data_.data());
+    }
+
+    char*
+    cdata()
+    {
+        return reinterpret_cast<char*>(data_.data());
+    }
+
     iterator
     begin()
     {
@@ -549,8 +561,8 @@ using uint128 = base_uint<128>;
 using uint160 = base_uint<160>;
 using uint256 = base_uint<256>;
 
-/*
- * template <std::size_t Bits, class Tag>
+#ifdef __APPLE__
+template <std::size_t Bits, class Tag>
 [[nodiscard]] inline constexpr std::strong_ordering
 operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
 {
@@ -561,7 +573,6 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
     //
     // FIXME: use std::lexicographical_compare_three_way once support is
     //        added to MacOS.
-
     auto const ret = std::mismatch(lhs.cbegin(), lhs.cend(), rhs.cbegin());
 
     // a == b
@@ -571,8 +582,7 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
     return (*ret.first > *ret.second) ? std::strong_ordering::greater
                                       : std::strong_ordering::less;
 }
-*/
-
+#else
 template <std::size_t Bits, class Tag>
 [[nodiscard]] inline constexpr std::strong_ordering
 operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
@@ -584,6 +594,7 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
         rhs.cend(),
         std::compare_three_way{});
 }
+#endif
 
 template <std::size_t Bits, typename Tag>
 [[nodiscard]] inline constexpr bool

src/ripple/basics/impl/IOUAmount.cpp

@@ -27,7 +27,28 @@
 
 namespace ripple {
 
-LocalValue<bool> stNumberSwitchover(true);
+namespace {
+
+// Use a static inside a function to help prevent order-of-initialzation issues
+LocalValue<bool>&
+getStaticSTNumberSwitchover()
+{
+    static LocalValue<bool> r{true};
+    return r;
+}
+}  // namespace
+
+bool
+getSTNumberSwitchover()
+{
+    return *getStaticSTNumberSwitchover();
+}
+
+void
+setSTNumberSwitchover(bool v)
+{
+    *getStaticSTNumberSwitchover() = v;
+}
 
 /* The range for the mantissa when normalized */
 static std::int64_t constexpr minMantissa = 1000000000000000ull;
@@ -51,7 +72,7 @@ IOUAmount::normalize()
         return;
     }
 
-    if (*stNumberSwitchover)
+    if (getSTNumberSwitchover())
     {
         Number const v{mantissa_, exponent_};
         mantissa_ = v.mantissa();
@@ -117,7 +138,7 @@ IOUAmount::operator+=(IOUAmount const& other)
         return *this;
     }
 
-    if (*stNumberSwitchover)
+    if (getSTNumberSwitchover())
     {
         *this = IOUAmount{Number{*this} + Number{other}};
         return *this;

src/ripple/basics/impl/partitioned_unordered_map.cpp

@@ -31,7 +31,11 @@ namespace ripple {
 static std::size_t
 extract(uint256 const& key)
 {
-    return *reinterpret_cast<std::size_t const*>(key.data());
+    std::size_t result;
+    // Use memcpy to avoid unaligned UB
+    // (will optimize to equivalent code)
+    std::memcpy(&result, key.data(), sizeof(std::size_t));
+    return result;
 }
 
 static std::size_t

src/ripple/beast/container/detail/aged_ordered_container.h

@@ -145,111 +145,78 @@ private:
     };
 
     // VFALCO TODO This should only be enabled for maps.
-    class pair_value_compare
-        : public beast::detail::empty_base_optimization<Compare>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<value_type, value_type, bool>
-#endif
+    class pair_value_compare : public Compare
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument = value_type;
         using second_argument = value_type;
         using result_type = bool;
-#endif
 
         bool
         operator()(value_type const& lhs, value_type const& rhs) const
         {
-            return this->member()(lhs.first, rhs.first);
+            return Compare::operator()(lhs.first, rhs.first);
         }
 
         pair_value_compare()
         {
         }
 
-        pair_value_compare(pair_value_compare const& other)
-            : beast::detail::empty_base_optimization<Compare>(other)
+        pair_value_compare(pair_value_compare const& other) : Compare(other)
         {
         }
 
     private:
         friend aged_ordered_container;
 
-        pair_value_compare(Compare const& compare)
-            : beast::detail::empty_base_optimization<Compare>(compare)
+        pair_value_compare(Compare const& compare) : Compare(compare)
         {
         }
     };
 
     // Compares value_type against element, used in insert_check
     // VFALCO TODO hoist to remove template argument dependencies
-    class KeyValueCompare
-        : public beast::detail::empty_base_optimization<Compare>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<Key, element, bool>
-#endif
+    class KeyValueCompare : public Compare
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument = Key;
         using second_argument = element;
         using result_type = bool;
-#endif
 
         KeyValueCompare() = default;
 
-        KeyValueCompare(Compare const& compare)
-            : beast::detail::empty_base_optimization<Compare>(compare)
+        KeyValueCompare(Compare const& compare) : Compare(compare)
         {
         }
 
-        // VFALCO NOTE WE might want only to enable these overloads
-        //                if Compare has is_transparent
-#if 0
-        template <class K>
-        bool operator() (K const& k, element const& e) const
-        {
-            return this->member() (k, extract (e.value));
-        }
-
-        template <class K>
-        bool operator() (element const& e, K const& k) const
-        {
-            return this->member() (extract (e.value), k);
-        }
-#endif
-
         bool
         operator()(Key const& k, element const& e) const
         {
-            return this->member()(k, extract(e.value));
+            return Compare::operator()(k, extract(e.value));
         }
 
         bool
         operator()(element const& e, Key const& k) const
         {
-            return this->member()(extract(e.value), k);
+            return Compare::operator()(extract(e.value), k);
         }
 
         bool
         operator()(element const& x, element const& y) const
         {
-            return this->member()(extract(x.value), extract(y.value));
+            return Compare::operator()(extract(x.value), extract(y.value));
         }
 
         Compare&
         compare()
         {
-            return beast::detail::empty_base_optimization<Compare>::member();
+            return *this;
         }
 
         Compare const&
         compare() const
         {
-            return beast::detail::empty_base_optimization<Compare>::member();
+            return *this;
         }
     };
 

src/ripple/beast/container/detail/aged_unordered_container.h

@@ -148,115 +148,84 @@ private:
     };
 
     // VFALCO TODO hoist to remove template argument dependencies
-    class ValueHash : private beast::detail::empty_base_optimization<Hash>
-#ifdef _LIBCPP_VERSION
-        ,
-                      public std::unary_function<element, std::size_t>
-#endif
+    class ValueHash : public Hash
     {
     public:
-#ifndef _LIBCPP_VERSION
         using argument_type = element;
         using result_type = size_t;
-#endif
 
         ValueHash()
         {
         }
 
-        ValueHash(Hash const& h)
-            : beast::detail::empty_base_optimization<Hash>(h)
+        ValueHash(Hash const& h) : Hash(h)
         {
         }
 
         std::size_t
         operator()(element const& e) const
         {
-            return this->member()(extract(e.value));
+            return Hash::operator()(extract(e.value));
         }
 
         Hash&
         hash_function()
         {
-            return this->member();
+            return *this;
         }
 
         Hash const&
         hash_function() const
         {
-            return this->member();
+            return *this;
         }
     };
 
     // Compares value_type against element, used in find/insert_check
     // VFALCO TODO hoist to remove template argument dependencies
-    class KeyValueEqual
-        : private beast::detail::empty_base_optimization<KeyEqual>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<Key, element, bool>
-#endif
+    class KeyValueEqual : public KeyEqual
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument_type = Key;
         using second_argument_type = element;
         using result_type = bool;
-#endif
 
         KeyValueEqual()
         {
         }
 
-        KeyValueEqual(KeyEqual const& keyEqual)
-            : beast::detail::empty_base_optimization<KeyEqual>(keyEqual)
+        KeyValueEqual(KeyEqual const& keyEqual) : KeyEqual(keyEqual)
         {
         }
 
-        // VFALCO NOTE WE might want only to enable these overloads
-        //                if KeyEqual has is_transparent
-#if 0
-        template <class K>
-        bool operator() (K const& k, element const& e) const
-        {
-            return this->member() (k, extract (e.value));
-        }
-
-        template <class K>
-        bool operator() (element const& e, K const& k) const
-        {
-            return this->member() (extract (e.value), k);
-        }
-#endif
-
         bool
         operator()(Key const& k, element const& e) const
         {
-            return this->member()(k, extract(e.value));
+            return KeyEqual::operator()(k, extract(e.value));
         }
 
         bool
         operator()(element const& e, Key const& k) const
         {
-            return this->member()(extract(e.value), k);
+            return KeyEqual::operator()(extract(e.value), k);
         }
 
         bool
         operator()(element const& lhs, element const& rhs) const
         {
-            return this->member()(extract(lhs.value), extract(rhs.value));
+            return KeyEqual::operator()(extract(lhs.value), extract(rhs.value));
         }
 
         KeyEqual&
         key_eq()
         {
-            return this->member();
+            return *this;
         }
 
         KeyEqual const&
         key_eq() const
         {
-            return this->member();
+            return *this;
         }
     };
 

src/ripple/beast/hash/impl/xxhash.cpp

@@ -33,6 +33,8 @@ You can contact the author at :
 
 #include <ripple/beast/hash/impl/xxhash.h>
 
+#include <cstring>
+
 //**************************************
 // Tuning parameters
 //**************************************
@@ -87,7 +89,7 @@ You can contact the author at :
 //**************************************
 // Includes & Memory related functions
 //**************************************
-//#include "xxhash.h"
+// #include "xxhash.h"
 // Modify the local functions below should you wish to use some other memory
 // routines for malloc(), free()
 #include <stdlib.h>
@@ -260,7 +262,13 @@ FORCE_INLINE U64
 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align == XXH_unaligned)
-        return endian == XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr));
+    {
+        // Use memcpy to avoid unaligned UB
+        U64 tmp_aligned;
+        std::memcpy(&tmp_aligned, ptr, sizeof(U64));
+        return endian == XXH_littleEndian ? tmp_aligned
+                                          : XXH_swap64(tmp_aligned);
+    }
     else
         return endian == XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr);
 }

src/ripple/consensus/Consensus.h

@@ -1644,7 +1644,7 @@ Consensus<Adaptor>::createDisputes(TxSet_t const& o)
             (inThisSet && result_->txns.find(txId) && !o.find(txId)) ||
             (!inThisSet && !result_->txns.find(txId) && o.find(txId)));
 
-        Tx_t tx = inThisSet ? *result_->txns.find(txId) : *o.find(txId);
+        Tx_t tx = inThisSet ? result_->txns.find(txId) : o.find(txId);
         auto txID = tx.id();
 
         if (result_->disputes.find(txID) != result_->disputes.end())

src/ripple/core/Config.h

@@ -25,6 +25,7 @@
 #include <ripple/basics/base_uint.h>
 #include <ripple/beast/net/IPEndpoint.h>
 #include <ripple/beast/utility/Journal.h>
+#include <ripple/core/ConfigSections.h>
 #include <ripple/protocol/PublicKey.h>
 #include <ripple/protocol/SystemParameters.h>  // VFALCO Breaks levelization
 #include <boost/beast/core/string.hpp>
@@ -154,6 +155,8 @@ public:
     std::map<std::string, PublicKey>
         IMPORT_VL_KEYS;  // hex string -> class PublicKey (for caching purposes)
 
+    std::vector<std::string> DATAGRAM_MONITOR;
+
     enum StartUpType {
         FRESH,
         NORMAL,
@@ -350,6 +353,21 @@ public:
     {
         return RUN_REPORTING;
     }
+    bool
+    mem_backend() const
+    {
+        static bool const isMem =
+            (!section(SECTION_RELATIONAL_DB).empty() &&
+             boost::beast::iequals(
+                 get(section(SECTION_RELATIONAL_DB), "backend"), "rwdb")) ||
+            (!section("node_db").empty() &&
+             (boost::beast::iequals(get(section("node_db"), "type"), "rwdb") ||
+              boost::beast::iequals(
+                  get(section("node_db"), "type"), "flatmap")));
+        // RHNOTE: memory type is not selected for here because it breaks
+        // tests
+        return isMem;
+    }
 
     bool
     useTxTables() const

src/ripple/core/ConfigSections.h

@@ -101,6 +101,7 @@ struct ConfigSection
 #define SECTION_SWEEP_INTERVAL "sweep_interval"
 #define SECTION_NETWORK_ID "network_id"
 #define SECTION_IMPORT_VL_KEYS "import_vl_keys"
+#define SECTION_DATAGRAM_MONITOR "datagram_monitor"
 
 }  // namespace ripple
 

src/ripple/core/impl/Config.cpp

@@ -68,10 +68,8 @@ namespace detail {
 [[nodiscard]] std::uint64_t
 getMemorySize()
 {
-    struct sysinfo si;
-
-    if (sysinfo(&si) == 0)
-        return static_cast<std::uint64_t>(si.totalram);
+    if (struct sysinfo si; sysinfo(&si) == 0)
+        return static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
 
     return 0;
 }
@@ -265,7 +263,8 @@ getEnvVar(char const* name)
 }
 
 Config::Config()
-    : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize())
+    : j_(beast::Journal::getNullSink())
+    , ramSize_(detail::getMemorySize() / (1024 * 1024 * 1024))
 {
 }
 
@@ -282,6 +281,9 @@ Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
     // RAM and CPU resources. We default to "tiny" for standalone mode.
     if (!bStandalone)
     {
+        NODE_SIZE = 4;
+        return;
+
         // First, check against 'minimum' RAM requirements per node size:
         auto const& threshold =
             sizedItems[std::underlying_type_t<SizedItem>(SizedItem::ramSizeGB)];
@@ -290,22 +292,18 @@ Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
             threshold.second.begin(),
             threshold.second.end(),
             [this](std::size_t limit) {
-                return (ramSize_ / (1024 * 1024 * 1024)) < limit;
+                return (limit == 0) || (ramSize_ < limit);
             });
 
+        assert(ns != threshold.second.end());
+
         if (ns != threshold.second.end())
             NODE_SIZE = std::distance(threshold.second.begin(), ns);
 
         // Adjust the size based on the number of hardware threads of
         // execution available to us:
-        if (auto const hc = std::thread::hardware_concurrency())
-        {
-            if (hc == 1)
-                NODE_SIZE = 0;
-
-            if (hc < 4)
-                NODE_SIZE = std::min<std::size_t>(NODE_SIZE, 1);
-        }
+        if (auto const hc = std::thread::hardware_concurrency(); hc != 0)
+            NODE_SIZE = std::min<std::size_t>(hc / 2, NODE_SIZE);
     }
 
     assert(NODE_SIZE <= 4);
@@ -470,26 +468,24 @@ Config::loadFromString(std::string const& fileContents)
         SNTP_SERVERS = *s;
 
     // if the user has specified ip:port then replace : with a space.
-    {
-        auto replaceColons = [](std::vector<std::string>& strVec) {
-            const static std::regex e(":([0-9]+)$");
-            for (auto& line : strVec)
-            {
-                // skip anything that might be an ipv6 address
-                if (std::count(line.begin(), line.end(), ':') != 1)
-                    continue;
+    auto replaceColons = [](std::vector<std::string>& strVec) {
+        const static std::regex e(":([0-9]+)$");
+        for (auto& line : strVec)
+        {
+            // skip anything that might be an ipv6 address
+            if (std::count(line.begin(), line.end(), ':') != 1)
+                continue;
 
-                std::string result = std::regex_replace(line, e, " $1");
-                // sanity check the result of the replace, should be same length
-                // as input
-                if (result.size() == line.size())
-                    line = result;
-            }
-        };
+            std::string result = std::regex_replace(line, e, " $1");
+            // sanity check the result of the replace, should be same length
+            // as input
+            if (result.size() == line.size())
+                line = result;
+        }
+    };
 
-        replaceColons(IPS_FIXED);
-        replaceColons(IPS);
-    }
+    replaceColons(IPS_FIXED);
+    replaceColons(IPS);
 
     {
         std::string dbPath;
@@ -514,6 +510,12 @@ Config::loadFromString(std::string const& fileContents)
             NETWORK_ID = beast::lexicalCastThrow<uint32_t>(strTemp);
     }
 
+    if (auto s = getIniFileSection(secConfig, SECTION_DATAGRAM_MONITOR))
+    {
+        DATAGRAM_MONITOR = *s;
+        replaceColons(DATAGRAM_MONITOR);
+    }
+
     if (getSingleSection(secConfig, SECTION_PEER_PRIVATE, strTemp, j_))
         PEER_PRIVATE = beast::lexicalCastThrow<bool>(strTemp);
 

src/ripple/json/impl/json_reader.cpp

@@ -19,8 +19,10 @@
 
 #include <ripple/basics/contract.h>
 #include <ripple/json/json_reader.h>
+
 #include <algorithm>
 #include <cctype>
+#include <cstdint>
 #include <istream>
 #include <string>
 

src/ripple/net/RPCErr.h

@@ -28,7 +28,7 @@ namespace ripple {
 bool
 isRpcError(Json::Value jvResult);
 Json::Value
-rpcError(int iError);
+rpcError(int iError, std::string msg = "");
 
 }  // namespace ripple
 

src/ripple/net/ShardDownloader.md

@@ -195,7 +195,7 @@ three database entries upon completion.
 Since downloads execute serially by design, the entries in this table always
 correspond to the contents of a single file.
 
-| Bytes  | Size       | Part |
+| Bytes  | size       | Part |
 |:------:|:----------:|:----:|
 | 0x...  | 2147483647 | 0    |
 | 0x...  | 2147483647 | 1    |

src/ripple/net/impl/RPCCall.cpp

@@ -823,11 +823,9 @@ private:
         return jvRequest;
     }
 
-    // owner_info <account>|<account_public_key> [strict]
-    // owner_info <seed>|<pass_phrase>|<key> [<ledger>] [strict]
-    // account_info <account>|<account_public_key> [strict]
-    // account_info <seed>|<pass_phrase>|<key> [<ledger>] [strict]
-    // account_offers <account>|<account_public_key> [<ledger>] [strict]
+    // owner_info <account>
+    // account_info <account> [<ledger>]
+    // account_offers <account> [<ledger>]
     Json::Value
     parseAccountItems(Json::Value const& jvParams)
     {
@@ -1050,10 +1048,7 @@ private:
             // Parameters 0 and 1 are accounts
             if (i < 2)
             {
-                if (parseBase58<PublicKey>(
-                        TokenType::AccountPublic, strParam) ||
-                    parseBase58<AccountID>(strParam) ||
-                    parseGenericSeed(strParam))
+                if (parseBase58<AccountID>(strParam))
                 {
                     jvRequest[accFields[i]] = std::move(strParam);
                 }
@@ -1079,16 +1074,8 @@ private:
     {
         std::string strIdent = jvParams[0u].asString();
         unsigned int iCursor = jvParams.size();
-        bool bStrict = false;
 
-        if (iCursor >= 2 && jvParams[iCursor - 1] == jss::strict)
-        {
-            bStrict = true;
-            --iCursor;
-        }
-
-        if (!parseBase58<PublicKey>(TokenType::AccountPublic, strIdent) &&
-            !parseBase58<AccountID>(strIdent) && !parseGenericSeed(strIdent))
+        if (!parseBase58<AccountID>(strIdent))
             return rpcError(rpcACT_MALFORMED);
 
         // Get info on account.
@@ -1096,9 +1083,6 @@ private:
 
         jvRequest[jss::account] = strIdent;
 
-        if (bStrict)
-            jvRequest[jss::strict] = 1;
-
         if (iCursor == 2 && !jvParseLedger(jvRequest, jvParams[1u].asString()))
             return rpcError(rpcLGR_IDX_MALFORMED);
 
@@ -1894,7 +1878,9 @@ fromNetwork(
     constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256);
 
     using namespace std::chrono_literals;
-    auto constexpr RPC_NOTIFY = 10min;
+    // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one
+    // HTTP call?
+    auto constexpr RPC_NOTIFY = 30s;
 
     HTTPClient::request(
         bSSL,

src/ripple/net/impl/RPCErr.cpp

@@ -26,10 +26,14 @@ struct RPCErr;
 
 // VFALCO NOTE Deprecated function
 Json::Value
-rpcError(int iError)
+rpcError(int iError, std::string msg)
 {
     Json::Value jvResult(Json::objectValue);
-    RPC::inject_error(iError, jvResult);
+    if (msg != "")
+        RPC::inject_error(static_cast<error_code_i>(iError), msg, jvResult);
+    else
+        RPC::inject_error(iError, jvResult);
+
     return jvResult;
 }
 

src/ripple/net/impl/RPCSub.cpp

@@ -78,12 +78,14 @@ public:
     {
         std::lock_guard sl(mLock);
 
-        if (mDeque.size() >= eventQueueMax)
-        {
-            // Drop the previous event.
-            JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
-            mDeque.pop_back();
-        }
+        // Wietse: we're not going to limit this, this is admin-port only, scale
+        // accordingly Dropping events just like this results in inconsistent
+        // data on the receiving end if (mDeque.size() >= eventQueueMax)
+        // {
+        //     // Drop the previous event.
+        //     JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
+        //     mDeque.pop_back();
+        // }
 
         auto jm = broadcast ? j_.debug() : j_.info();
         JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj;
@@ -182,7 +184,8 @@ private:
     }
 
 private:
-    enum { eventQueueMax = 32 };
+    // Wietse: we're not going to limit this, this is admin-port only, scale
+    // accordingly enum { eventQueueMax = 32 };
 
     boost::asio::io_service& m_io_service;
     JobQueue& m_jobQueue;

src/ripple/nodestore/DeterministicShard.md

@@ -22,7 +22,7 @@ uint64          Appnum          Application defined constant
 uint16          KeySize         Key size in bytes
 uint64          Salt            A random seed
 uint64          Pepper          The salt hashed
-uint16          BlockSize       Size of a file block in bytes
+uint16          BlockSize       size of a file block in bytes
 uint16          LoadFactor      Target fraction in 65536ths
 uint8[56]       Reserved        Zeroes
 uint8[]         Reserved        Zero-pad to block size
@@ -160,4 +160,3 @@ Iteration 0: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71
 Iteration 1: RIPEMD160[nudb.key] = F96BF2722AB2EE009FFAE4A36AAFC4F220E21951
 Iteration 1: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71
 ```
-

src/ripple/nodestore/ShardSizeTuning.md

@@ -1,4 +1,4 @@
-# Shard Size Tuning
+# Shard size Tuning
 
 The purpose of this document is to compare the sizes of shards containing
 varying amounts of ledgers.

src/ripple/nodestore/backend/FlatmapFactory.cpp

@@ -0,0 +1,235 @@
+#include <ripple/basics/contract.h>
+#include <ripple/nodestore/Factory.h>
+#include <ripple/nodestore/Manager.h>
+#include <ripple/nodestore/impl/DecodedBlob.h>
+#include <ripple/nodestore/impl/EncodedBlob.h>
+#include <ripple/nodestore/impl/codec.h>
+#include <boost/beast/core/string.hpp>
+#include <boost/core/ignore_unused.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
+#include <memory>
+#include <mutex>
+
+namespace ripple {
+namespace NodeStore {
+
+class FlatmapBackend : public Backend
+{
+private:
+    std::string name_;
+    beast::Journal journal_;
+    bool isOpen_{false};
+
+    struct base_uint_hasher
+    {
+        using result_type = std::size_t;
+
+        result_type
+        operator()(base_uint<256> const& value) const
+        {
+            return hardened_hash<>{}(value);
+        }
+    };
+
+    using DataStore = boost::unordered::concurrent_flat_map<
+        uint256,
+        std::vector<std::uint8_t>,  // Store compressed blob data
+        base_uint_hasher>;
+
+    DataStore table_;
+
+public:
+    FlatmapBackend(
+        size_t keyBytes,
+        Section const& keyValues,
+        beast::Journal journal)
+        : name_(get(keyValues, "path")), journal_(journal)
+    {
+        boost::ignore_unused(journal_);
+        if (name_.empty())
+            name_ = "node_db";
+    }
+
+    ~FlatmapBackend() override
+    {
+        close();
+    }
+
+    std::string
+    getName() override
+    {
+        return name_;
+    }
+
+    void
+    open(bool createIfMissing) override
+    {
+        if (isOpen_)
+            Throw<std::runtime_error>("already open");
+        isOpen_ = true;
+    }
+
+    bool
+    isOpen() override
+    {
+        return isOpen_;
+    }
+
+    void
+    close() override
+    {
+        table_.clear();
+        isOpen_ = false;
+    }
+
+    Status
+    fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
+    {
+        if (!isOpen_)
+            return notFound;
+
+        uint256 const hash(uint256::fromVoid(key));
+
+        bool found = table_.visit(hash, [&](const auto& key_value_pair) {
+            nudb::detail::buffer bf;
+            auto const result = nodeobject_decompress(
+                key_value_pair.second.data(), key_value_pair.second.size(), bf);
+            DecodedBlob decoded(hash.data(), result.first, result.second);
+            if (!decoded.wasOk())
+            {
+                *pObject = nullptr;
+                return;
+            }
+            *pObject = decoded.createObject();
+        });
+        return found ? (*pObject ? ok : dataCorrupt) : notFound;
+    }
+
+    std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
+    fetchBatch(std::vector<uint256 const*> const& hashes) override
+    {
+        std::vector<std::shared_ptr<NodeObject>> results;
+        results.reserve(hashes.size());
+        for (auto const& h : hashes)
+        {
+            std::shared_ptr<NodeObject> nObj;
+            Status status = fetch(h->begin(), &nObj);
+            if (status != ok)
+                results.push_back({});
+            else
+                results.push_back(nObj);
+        }
+        return {results, ok};
+    }
+
+    void
+    store(std::shared_ptr<NodeObject> const& object) override
+    {
+        if (!isOpen_)
+            return;
+
+        if (!object)
+            return;
+
+        EncodedBlob encoded(object);
+        nudb::detail::buffer bf;
+        auto const result =
+            nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
+
+        std::vector<std::uint8_t> compressed(
+            static_cast<const std::uint8_t*>(result.first),
+            static_cast<const std::uint8_t*>(result.first) + result.second);
+
+        table_.insert_or_assign(object->getHash(), std::move(compressed));
+    }
+
+    void
+    storeBatch(Batch const& batch) override
+    {
+        for (auto const& e : batch)
+            store(e);
+    }
+
+    void
+    sync() override
+    {
+    }
+
+    void
+    for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
+    {
+        if (!isOpen_)
+            return;
+
+        table_.visit_all([&f](const auto& entry) {
+            nudb::detail::buffer bf;
+            auto const result = nodeobject_decompress(
+                entry.second.data(), entry.second.size(), bf);
+            DecodedBlob decoded(
+                entry.first.data(), result.first, result.second);
+            if (decoded.wasOk())
+                f(decoded.createObject());
+        });
+    }
+
+    int
+    getWriteLoad() override
+    {
+        return 0;
+    }
+
+    void
+    setDeletePath() override
+    {
+        close();
+    }
+
+    int
+    fdRequired() const override
+    {
+        return 0;
+    }
+
+private:
+    size_t
+    size() const
+    {
+        return table_.size();
+    }
+};
+
+class FlatmapFactory : public Factory
+{
+public:
+    FlatmapFactory()
+    {
+        Manager::instance().insert(*this);
+    }
+
+    ~FlatmapFactory() override
+    {
+        Manager::instance().erase(*this);
+    }
+
+    std::string
+    getName() const override
+    {
+        return "Flatmap";
+    }
+
+    std::unique_ptr<Backend>
+    createInstance(
+        size_t keyBytes,
+        Section const& keyValues,
+        std::size_t burstSize,
+        Scheduler& scheduler,
+        beast::Journal journal) override
+    {
+        return std::make_unique<FlatmapBackend>(keyBytes, keyValues, journal);
+    }
+};
+
+static FlatmapFactory flatmapFactory;
+
+}  // namespace NodeStore
+}  // namespace ripple

src/ripple/nodestore/backend/MemoryFactory.cpp

@@ -94,7 +94,7 @@ public:
     {
         boost::ignore_unused(journal_);  // Keep unused journal_ just in case.
         if (name_.empty())
-            Throw<std::runtime_error>("Missing path in Memory backend");
+            Throw<std::runtime_error>("Missing path in TestMemory backend");
     }
 
     ~MemoryBackend() override

src/ripple/nodestore/backend/RWDBFactory.cpp

@@ -0,0 +1,242 @@
+#include <ripple/basics/contract.h>
+#include <ripple/nodestore/Factory.h>
+#include <ripple/nodestore/Manager.h>
+#include <ripple/nodestore/impl/DecodedBlob.h>
+#include <ripple/nodestore/impl/EncodedBlob.h>
+#include <ripple/nodestore/impl/codec.h>
+#include <boost/beast/core/string.hpp>
+#include <boost/core/ignore_unused.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
+#include <memory>
+#include <mutex>
+
+namespace ripple {
+namespace NodeStore {
+
+class RWDBBackend : public Backend
+{
+private:
+    std::string name_;
+    beast::Journal journal_;
+    bool isOpen_{false};
+
+    struct base_uint_hasher
+    {
+        using result_type = std::size_t;
+
+        result_type
+        operator()(base_uint<256> const& value) const
+        {
+            return hardened_hash<>{}(value);
+        }
+    };
+
+    using DataStore =
+        std::map<uint256, std::vector<std::uint8_t>>;  // Store compressed blob
+                                                       // data
+    mutable std::recursive_mutex
+        mutex_;  // Only needed for std::map implementation
+
+    DataStore table_;
+
+public:
+    RWDBBackend(
+        size_t keyBytes,
+        Section const& keyValues,
+        beast::Journal journal)
+        : name_(get(keyValues, "path")), journal_(journal)
+    {
+        boost::ignore_unused(journal_);
+        if (name_.empty())
+            name_ = "node_db";
+    }
+
+    ~RWDBBackend() override
+    {
+        close();
+    }
+
+    std::string
+    getName() override
+    {
+        return name_;
+    }
+
+    void
+    open(bool createIfMissing) override
+    {
+        std::lock_guard lock(mutex_);
+        if (isOpen_)
+            Throw<std::runtime_error>("already open");
+        isOpen_ = true;
+    }
+
+    bool
+    isOpen() override
+    {
+        return isOpen_;
+    }
+
+    void
+    close() override
+    {
+        std::lock_guard lock(mutex_);
+        table_.clear();
+        isOpen_ = false;
+    }
+
+    Status
+    fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
+    {
+        if (!isOpen_)
+            return notFound;
+
+        uint256 const hash(uint256::fromVoid(key));
+
+        std::lock_guard lock(mutex_);
+        auto it = table_.find(hash);
+        if (it == table_.end())
+            return notFound;
+
+        nudb::detail::buffer bf;
+        auto const result =
+            nodeobject_decompress(it->second.data(), it->second.size(), bf);
+        DecodedBlob decoded(hash.data(), result.first, result.second);
+        if (!decoded.wasOk())
+            return dataCorrupt;
+        *pObject = decoded.createObject();
+        return ok;
+    }
+
+    std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
+    fetchBatch(std::vector<uint256 const*> const& hashes) override
+    {
+        std::vector<std::shared_ptr<NodeObject>> results;
+        results.reserve(hashes.size());
+        for (auto const& h : hashes)
+        {
+            std::shared_ptr<NodeObject> nObj;
+            Status status = fetch(h->begin(), &nObj);
+            if (status != ok)
+                results.push_back({});
+            else
+                results.push_back(nObj);
+        }
+        return {results, ok};
+    }
+
+    void
+    store(std::shared_ptr<NodeObject> const& object) override
+    {
+        if (!isOpen_)
+            return;
+
+        if (!object)
+            return;
+
+        EncodedBlob encoded(object);
+        nudb::detail::buffer bf;
+        auto const result =
+            nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
+
+        std::vector<std::uint8_t> compressed(
+            static_cast<const std::uint8_t*>(result.first),
+            static_cast<const std::uint8_t*>(result.first) + result.second);
+
+        std::lock_guard lock(mutex_);
+        table_[object->getHash()] = std::move(compressed);
+    }
+
+    void
+    storeBatch(Batch const& batch) override
+    {
+        for (auto const& e : batch)
+            store(e);
+    }
+
+    void
+    sync() override
+    {
+    }
+
+    void
+    for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
+    {
+        if (!isOpen_)
+            return;
+
+        std::lock_guard lock(mutex_);
+        for (const auto& entry : table_)
+        {
+            nudb::detail::buffer bf;
+            auto const result = nodeobject_decompress(
+                entry.second.data(), entry.second.size(), bf);
+            DecodedBlob decoded(
+                entry.first.data(), result.first, result.second);
+            if (decoded.wasOk())
+                f(decoded.createObject());
+        }
+    }
+
+    int
+    getWriteLoad() override
+    {
+        return 0;
+    }
+
+    void
+    setDeletePath() override
+    {
+        close();
+    }
+
+    int
+    fdRequired() const override
+    {
+        return 0;
+    }
+
+private:
+    size_t
+    size() const
+    {
+        std::lock_guard lock(mutex_);
+        return table_.size();
+    }
+};
+
+class RWDBFactory : public Factory
+{
+public:
+    RWDBFactory()
+    {
+        Manager::instance().insert(*this);
+    }
+
+    ~RWDBFactory() override
+    {
+        Manager::instance().erase(*this);
+    }
+
+    std::string
+    getName() const override
+    {
+        return "RWDB";
+    }
+
+    std::unique_ptr<Backend>
+    createInstance(
+        size_t keyBytes,
+        Section const& keyValues,
+        std::size_t burstSize,
+        Scheduler& scheduler,
+        beast::Journal journal) override
+    {
+        return std::make_unique<RWDBBackend>(keyBytes, keyValues, journal);
+    }
+};
+
+static RWDBFactory rwDBFactory;
+
+}  // namespace NodeStore
+}  // namespace ripple

src/ripple/nodestore/impl/DatabaseRotatingImp.cpp

@@ -18,6 +18,8 @@
 //==============================================================================
 
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/ledger/LedgerMaster.h>
+#include <ripple/app/main/Application.h>
 #include <ripple/nodestore/impl/DatabaseRotatingImp.h>
 #include <ripple/protocol/HashPrefix.h>
 
@@ -25,6 +27,7 @@ namespace ripple {
 namespace NodeStore {
 
 DatabaseRotatingImp::DatabaseRotatingImp(
+    Application& app,
     Scheduler& scheduler,
     int readThreads,
     std::shared_ptr<Backend> writableBackend,
@@ -32,6 +35,7 @@ DatabaseRotatingImp::DatabaseRotatingImp(
     Section const& config,
     beast::Journal j)
     : DatabaseRotating(scheduler, readThreads, config, j)
+    , app_(app)
     , writableBackend_(std::move(writableBackend))
     , archiveBackend_(std::move(archiveBackend))
 {
@@ -48,8 +52,58 @@ DatabaseRotatingImp::rotateWithLock(
 {
     std::lock_guard lock(mutex_);
 
+    // Create the new backend
     auto newBackend = f(writableBackend_->getName());
+
+    // Before rotating, ensure all pinned ledgers are in the writable backend
+    JLOG(j_.info())
+        << "Ensuring pinned ledgers are preserved before backend rotation";
+
+    // Use a lambda to handle the preservation of pinned ledgers
+    auto ensurePinnedLedgersInWritable = [this]() {
+        // Get list of pinned ledgers
+        auto pinnedLedgers = app_.getLedgerMaster().getPinnedLedgersRangeSet();
+
+        for (auto const& range : pinnedLedgers)
+        {
+            for (auto seq = range.lower(); seq <= range.upper(); ++seq)
+            {
+                uint256 hash = app_.getLedgerMaster().getHashBySeq(seq);
+                if (hash.isZero())
+                    continue;
+
+                // Try to load the ledger
+                auto ledger = app_.getLedgerMaster().getLedgerByHash(hash);
+                if (ledger && ledger->isImmutable())
+                {
+                    // If we have the ledger, store it in the writable backend
+                    JLOG(j_.debug()) << "Ensuring pinned ledger " << seq
+                                     << " is in writable backend";
+                    Database::storeLedger(*ledger, writableBackend_);
+                }
+                else
+                {
+                    // If we don't have the ledger in memory, try to fetch its
+                    // objects directly
+                    JLOG(j_.debug()) << "Attempting to copy pinned ledger "
+                                     << seq << " header to writable backend";
+                    std::shared_ptr<NodeObject> headerObj;
+                    Status status =
+                        archiveBackend_->fetch(hash.data(), &headerObj);
+                    if (status == ok && headerObj)
+                        writableBackend_->store(headerObj);
+                }
+            }
+        }
+    };
+
+    // Execute the lambda
+    ensurePinnedLedgersInWritable();
+
+    // Now it's safe to mark the archive backend for deletion
     archiveBackend_->setDeletePath();
+
+    // Complete the rotation
     archiveBackend_ = std::move(writableBackend_);
     writableBackend_ = std::move(newBackend);
 }
@@ -180,8 +234,8 @@ DatabaseRotatingImp::fetchNodeObject(
             }
 
             // Update writable backend with data from the archive backend
-            if (duplicate)
-                writable->store(nodeObject);
+            // if (duplicate)
+            writable->store(nodeObject);
         }
     }
 

src/ripple/nodestore/impl/DatabaseRotatingImp.h

@@ -33,7 +33,10 @@ public:
     DatabaseRotatingImp&
     operator=(DatabaseRotatingImp const&) = delete;
 
+    Application& app_;
+
     DatabaseRotatingImp(
+        Application& app,
         Scheduler& scheduler,
         int readThreads,
         std::shared_ptr<Backend> writableBackend,

src/ripple/nodestore/impl/DecodedBlob.cpp

@@ -38,7 +38,6 @@ DecodedBlob::DecodedBlob(void const* key, void const* value, int valueBytes)
 
     m_success = false;
     m_key = key;
-    // VFALCO NOTE Ledger indexes should have started at 1
     m_objectType = hotUNKNOWN;
     m_objectData = nullptr;
     m_dataBytes = std::max(0, valueBytes - 9);

src/ripple/nodestore/impl/Shard.cpp

@@ -688,9 +688,6 @@ Shard::finalize(bool writeSQLite, std::optional<uint256> const& referenceHash)
 
         ledger->stateMap().setLedgerSeq(ledgerSeq);
         ledger->txMap().setLedgerSeq(ledgerSeq);
-        // RH TODO: investigate why this assertion was failing in
-        // ripple.rpc.NodeToShardRPC
-        // assert(ledger->read(keylet::fees()));
         ledger->setImmutable();
         if (!ledger->stateMap().fetchRoot(
                 SHAMapHash{ledger->info().accountHash}, nullptr))
@@ -713,6 +710,8 @@ Shard::finalize(bool writeSQLite, std::optional<uint256> const& referenceHash)
         if (writeSQLite && !storeSQLite(ledger))
             return fail("failed storing to SQLite databases");
 
+        assert(ledger->info().seq == ledgerSeq && ledger->read(keylet::fees()));
+
         hash = ledger->info().parentHash;
         next = std::move(ledger);
 

src/ripple/overlay/impl/OverlayImpl.cpp

@@ -38,6 +38,7 @@
 #include <ripple/rpc/json_body.h>
 #include <ripple/server/SimpleWriter.h>
 
+#include <ripple/core/ConfigSections.h>
 #include <boost/algorithm/string/predicate.hpp>
 #include <boost/utility/in_place_factory.hpp>
 
@@ -136,7 +137,11 @@ OverlayImpl::OverlayImpl(
           stopwatch(),
           app_.journal("PeerFinder"),
           config,
-          collector))
+          collector,
+          app.config().section(SECTION_RELATIONAL_DB).empty() ||
+              !boost::iequals(
+                  get(app.config().section(SECTION_RELATIONAL_DB), "backend"),
+                  "rwdb")))
     , m_resolver(resolver)
     , next_id_(1)
     , timer_count_(0)

src/ripple/peerfinder/impl/Bootcache.h

@@ -91,17 +91,10 @@ private:
     using value_type = map_type::value_type;
 
     struct Transform
-#ifdef _LIBCPP_VERSION
-        : std::unary_function<
-              map_type::right_map::const_iterator::value_type const&,
-              beast::IP::Endpoint const&>
-#endif
     {
-#ifndef _LIBCPP_VERSION
         using first_argument_type =
             map_type::right_map::const_iterator::value_type const&;
         using result_type = beast::IP::Endpoint const&;
-#endif
 
         explicit Transform() = default;
 

src/ripple/peerfinder/impl/InMemoryStore.h

@@ -0,0 +1,54 @@
+#ifndef RIPPLE_PEERFINDER_INMEMORYSTORE_H_INCLUDED
+#define RIPPLE_PEERFINDER_INMEMORYSTORE_H_INCLUDED
+
+#include <ripple/beast/net/IPEndpoint.h>
+#include <ripple/peerfinder/impl/Store.h>
+#include <boost/functional/hash.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
+
+namespace ripple {
+namespace PeerFinder {
+
+struct EndpointHasher
+{
+    std::size_t
+    operator()(beast::IP::Endpoint const& endpoint) const
+    {
+        std::size_t seed = 0;
+        boost::hash_combine(seed, endpoint.address().to_string());
+        boost::hash_combine(seed, endpoint.port());
+        return seed;
+    }
+};
+
+class InMemoryStore : public Store
+{
+private:
+    boost::concurrent_flat_map<beast::IP::Endpoint, int, EndpointHasher>
+        entries;
+
+public:
+    std::size_t
+    load(load_callback const& cb) override
+    {
+        std::size_t count = 0;
+        entries.visit_all([&](auto const& entry) {
+            cb(entry.first, entry.second);
+            ++count;
+        });
+        return count;
+    }
+
+    void
+    save(std::vector<Entry> const& v) override
+    {
+        entries.clear();
+        for (auto const& entry : v)
+            entries.emplace(entry.endpoint, entry.valence);
+    }
+};
+
+}  // namespace PeerFinder
+}  // namespace ripple
+
+#endif

src/ripple/peerfinder/impl/Livecache.h

@@ -69,14 +69,9 @@ public:
     public:
         // Iterator transformation to extract the endpoint from Element
         struct Transform
-#ifdef _LIBCPP_VERSION
-            : public std::unary_function<Element, Endpoint>
-#endif
         {
-#ifndef _LIBCPP_VERSION
             using first_argument = Element;
             using result_type = Endpoint;
-#endif
 
             explicit Transform() = default;
 
@@ -239,15 +234,9 @@ public:
 
         template <bool IsConst>
         struct Transform
-#ifdef _LIBCPP_VERSION
-            : public std::
-                  unary_function<typename lists_type::value_type, Hop<IsConst>>
-#endif
         {
-#ifndef _LIBCPP_VERSION
             using first_argument = typename lists_type::value_type;
             using result_type = Hop<IsConst>;
-#endif
 
             explicit Transform() = default;
 

src/ripple/peerfinder/impl/PeerfinderManager.cpp

@@ -17,8 +17,10 @@
 */
 //==============================================================================
 
+#include <ripple/core/ConfigSections.h>
 #include <ripple/peerfinder/PeerfinderManager.h>
 #include <ripple/peerfinder/impl/Checker.h>
+#include <ripple/peerfinder/impl/InMemoryStore.h>
 #include <ripple/peerfinder/impl/Logic.h>
 #include <ripple/peerfinder/impl/SourceStrings.h>
 #include <ripple/peerfinder/impl/StoreSqdb.h>
@@ -38,7 +40,7 @@ public:
     std::optional<boost::asio::io_service::work> work_;
     clock_type& m_clock;
     beast::Journal m_journal;
-    StoreSqdb m_store;
+    std::unique_ptr<Store> m_store;
     Checker<boost::asio::ip::tcp> checker_;
     Logic<decltype(checker_)> m_logic;
     BasicConfig const& m_config;
@@ -50,15 +52,18 @@ public:
         clock_type& clock,
         beast::Journal journal,
         BasicConfig const& config,
-        beast::insight::Collector::ptr const& collector)
+        beast::insight::Collector::ptr const& collector,
+        bool useSqLiteStore)
         : Manager()
         , io_service_(io_service)
         , work_(std::in_place, std::ref(io_service_))
         , m_clock(clock)
         , m_journal(journal)
-        , m_store(journal)
+        , m_store(
+              useSqLiteStore ? static_cast<Store*>(new StoreSqdb(journal))
+                             : static_cast<Store*>(new InMemoryStore()))
         , checker_(io_service_)
-        , m_logic(clock, m_store, checker_, journal)
+        , m_logic(clock, *m_store, checker_, journal)
         , m_config(config)
         , m_stats(std::bind(&ManagerImp::collect_metrics, this), collector)
     {
@@ -215,7 +220,8 @@ public:
     void
     start() override
     {
-        m_store.open(m_config);
+        if (auto sqdb = dynamic_cast<StoreSqdb*>(m_store.get()))
+            sqdb->open(m_config);
         m_logic.load();
     }
 
@@ -275,10 +281,11 @@ make_Manager(
     clock_type& clock,
     beast::Journal journal,
     BasicConfig const& config,
-    beast::insight::Collector::ptr const& collector)
+    beast::insight::Collector::ptr const& collector,
+    bool useSqLiteStore)
 {
     return std::make_unique<ManagerImp>(
-        io_service, clock, journal, config, collector);
+        io_service, clock, journal, config, collector, useSqLiteStore);
 }
 
 }  // namespace PeerFinder

src/ripple/peerfinder/make_Manager.h

@@ -34,7 +34,8 @@ make_Manager(
     clock_type& clock,
     beast::Journal journal,
     BasicConfig const& config,
-    beast::insight::Collector::ptr const& collector);
+    beast::insight::Collector::ptr const& collector,
+    bool useSqliteStore);
 
 }  // namespace PeerFinder
 }  // namespace ripple

src/ripple/protocol/ErrorCodes.h

@@ -143,8 +143,9 @@ enum error_code_i {
 
     rpcOBJECT_NOT_FOUND = 92,
 
-    rpcLAST =
-        rpcOBJECT_NOT_FOUND  // rpcLAST should always equal the last code.=
+    rpcLEDGER_MISSING = 93,
+
+    rpcLAST = rpcLEDGER_MISSING  // rpcLAST should always equal the last code.=
 };
 
 /** Codes returned in the `warnings` array of certain RPC commands.