[fix] github runner

Co-Authored-By: Niq Dudfield <[email protected]>

.github/workflows/build-in-docker.yml

@@ -14,7 +14,7 @@ jobs:
   checkout:
     runs-on: [self-hosted, vanity]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         clean: false
   checkpatterns:

.github/workflows/clang-format.yml

@@ -30,7 +30,7 @@ jobs:
         git diff --exit-code | tee "clang-format.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: clang-format.patch

.github/workflows/levelization.yml

@@ -18,7 +18,7 @@ jobs:
         git diff --exit-code | tee "levelization.patch"
     - name: Upload patch
       if: failure() && steps.assert.outcome == 'failure'
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       continue-on-error: true
       with:
         name: levelization.patch

.vscode/settings.json

@@ -3,7 +3,7 @@
   "C_Cpp.clang_format_path": ".clang-format",
   "C_Cpp.clang_format_fallbackStyle": "{ ColumnLimit: 0 }",
   "[cpp]":{
-    "editor.wordBasedSuggestions": false,
+    "editor.wordBasedSuggestions": "off",
     "editor.suggest.insertMode": "replace",
     "editor.semanticHighlighting.enabled": true,
     "editor.tabSize": 4,

Builds/CMake/RippledCore.cmake

@@ -392,6 +392,7 @@ target_sources (rippled PRIVATE
   src/ripple/app/misc/NegativeUNLVote.cpp
   src/ripple/app/misc/NetworkOPs.cpp
   src/ripple/app/misc/SHAMapStoreImp.cpp
+  src/ripple/app/misc/StateAccounting.cpp
   src/ripple/app/misc/detail/impl/WorkSSL.cpp
   src/ripple/app/misc/impl/AccountTxPaging.cpp
   src/ripple/app/misc/impl/AmendmentTable.cpp
@@ -433,13 +434,17 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/CancelOffer.cpp
   src/ripple/app/tx/impl/CashCheck.cpp
   src/ripple/app/tx/impl/Change.cpp
+  src/ripple/app/tx/impl/ClaimReward.cpp
   src/ripple/app/tx/impl/CreateCheck.cpp
   src/ripple/app/tx/impl/CreateOffer.cpp
   src/ripple/app/tx/impl/CreateTicket.cpp
   src/ripple/app/tx/impl/DeleteAccount.cpp
   src/ripple/app/tx/impl/DepositPreauth.cpp
   src/ripple/app/tx/impl/Escrow.cpp
+  src/ripple/app/tx/impl/GenesisMint.cpp
+  src/ripple/app/tx/impl/Import.cpp
   src/ripple/app/tx/impl/InvariantCheck.cpp
+  src/ripple/app/tx/impl/Invoke.cpp
   src/ripple/app/tx/impl/NFTokenAcceptOffer.cpp
   src/ripple/app/tx/impl/NFTokenBurn.cpp
   src/ripple/app/tx/impl/NFTokenCancelOffer.cpp
@@ -448,14 +453,10 @@ target_sources (rippled PRIVATE
   src/ripple/app/tx/impl/OfferStream.cpp
   src/ripple/app/tx/impl/PayChan.cpp
   src/ripple/app/tx/impl/Payment.cpp
-  src/ripple/app/tx/impl/SetAccount.cpp
-  src/ripple/app/tx/impl/SetRegularKey.cpp
-  src/ripple/app/tx/impl/SetHook.cpp
-  src/ripple/app/tx/impl/ClaimReward.cpp
-  src/ripple/app/tx/impl/GenesisMint.cpp
-  src/ripple/app/tx/impl/Import.cpp
-  src/ripple/app/tx/impl/Invoke.cpp
   src/ripple/app/tx/impl/Remit.cpp
+  src/ripple/app/tx/impl/SetAccount.cpp
+  src/ripple/app/tx/impl/SetHook.cpp
+  src/ripple/app/tx/impl/SetRegularKey.cpp
   src/ripple/app/tx/impl/SetSignerList.cpp
   src/ripple/app/tx/impl/SetTrust.cpp
   src/ripple/app/tx/impl/SignerEntries.cpp
@@ -538,7 +539,9 @@ target_sources (rippled PRIVATE
        subdir: nodestore
   #]===============================]
   src/ripple/nodestore/backend/CassandraFactory.cpp
+  src/ripple/nodestore/backend/RWDBFactory.cpp
   src/ripple/nodestore/backend/MemoryFactory.cpp
+  src/ripple/nodestore/backend/FlatmapFactory.cpp
   src/ripple/nodestore/backend/NuDBFactory.cpp
   src/ripple/nodestore/backend/NullFactory.cpp
   src/ripple/nodestore/backend/RocksDBFactory.cpp
@@ -669,6 +672,9 @@ target_sources (rippled PRIVATE
   src/ripple/rpc/impl/ShardVerificationScheduler.cpp
   src/ripple/rpc/impl/Status.cpp
   src/ripple/rpc/impl/TransactionSign.cpp
+  src/ripple/rpc/impl/NFTokenID.cpp
+  src/ripple/rpc/impl/NFTokenOfferID.cpp
+  src/ripple/rpc/impl/NFTSyntheticSerializer.cpp
   #[===============================[
      main sources:
        subdir: perflog

Builds/CMake/deps/Boost.cmake

@@ -44,7 +44,7 @@ else()
 endif()
 # TBD:
 # Boost_USE_DEBUG_RUNTIME:  When ON, uses Boost libraries linked against the
-find_package(Boost 1.70 REQUIRED
+find_package(Boost 1.86 REQUIRED
   COMPONENTS
     chrono
     container

Builds/levelization/results/loops.txt

@@ -53,7 +53,7 @@ Loop: test.app test.jtx
   test.app > test.jtx
 
 Loop: test.app test.rpc
-  test.rpc == test.app
+  test.rpc ~= test.app
 
 Loop: test.jtx test.toplevel
   test.toplevel > test.jtx

README.md

@@ -1,4 +1,4 @@
-# Xahau
+# Xahau 
 
 **Note:** Throughout this README, references to "we" or "our" pertain to the community and contributors involved in the Xahau network. It does not imply a legal entity or a specific collection of individuals.
 
@@ -67,5 +67,5 @@ git-subtree. See those directories' README files for more details.
   - [explorer.xahau.network](https://explorer.xahau.network)
 - **Testnet & Faucet**: Test applications and obtain test XAH at [xahau-test.net](https://xahau-test.net) and use the testnet explorer at [explorer.xahau.network](https://explorer.xahau.network).
 - **Supporting Wallets**: A list of wallets that support XAH and Xahau-based assets.
-  - [Xumm](https://xumm.app)
-  - [Crossmark](https://crossmark.io)
+  - [Xaman](https://xaman.app)
+  - [Crossmark](https://crossmark.io)

build-core.sh

@@ -1,4 +1,11 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
+
+set -e
 
 echo "START INSIDE CONTAINER - CORE"
 
@@ -23,7 +30,7 @@ fi
 perl -i -pe "s/^(\\s*)-DBUILD_SHARED_LIBS=OFF/\\1-DBUILD_SHARED_LIBS=OFF\\n\\1-DROCKSDB_BUILD_SHARED=OFF/g" Builds/CMake/deps/Rocksdb.cmake &&
 mv Builds/CMake/deps/WasmEdge.cmake Builds/CMake/deps/WasmEdge.old &&
 echo "find_package(LLVM REQUIRED CONFIG)
-message(STATUS \"Found LLVM ${LLVM_PACKAGE_VERSION}\")
+message(STATUS \"Found LLVM \${LLVM_PACKAGE_VERSION}\")
 message(STATUS \"Using LLVMConfig.cmake in: \${LLVM_DIR}\")
 add_library (wasmedge STATIC IMPORTED GLOBAL)
 set_target_properties(wasmedge PROPERTIES IMPORTED_LOCATION \${WasmEdge_LIB})

build-full.sh

@@ -1,4 +1,11 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
+
+set -e
 
 echo "START INSIDE CONTAINER - FULL"
 
@@ -19,7 +26,7 @@ yum-config-manager --disable centos-sclo-sclo
 ####
 
 cd /io;
-mkdir src/certs;
+mkdir -p src/certs;
 curl --silent -k https://raw.githubusercontent.com/RichardAH/rippled-release-builder/main/ca-bundle/certbundle.h -o src/certs/certbundle.h;
 if [ "`grep certbundle.h src/ripple/net/impl/RegisterSSLCerts.cpp | wc -l`" -eq "0" ]
 then
@@ -66,8 +73,8 @@ then
     #endif/g" src/ripple/net/impl/RegisterSSLCerts.cpp &&
     sed -i "s/#include <ripple\/net\/RegisterSSLCerts.h>/\0\n#include <certs\/certbundle.h>/g" src/ripple/net/impl/RegisterSSLCerts.cpp
 fi
-mkdir .nih_c;
-mkdir .nih_toolchain;
+mkdir -p .nih_c;
+mkdir -p .nih_toolchain;
 cd .nih_toolchain &&
 yum install -y wget lz4 lz4-devel git llvm13-static.x86_64 llvm13-devel.x86_64 devtoolset-10-binutils zlib-static ncurses-static -y \
   devtoolset-7-gcc-c++ \
@@ -90,11 +97,11 @@ echo "-- Install Cmake 3.23.1 --" &&
 pwd &&
 ( wget -nc -q https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-linux-x86_64.tar.gz; echo "" ) &&
 tar -xzf cmake-3.23.1-linux-x86_64.tar.gz -C /hbb/ &&
-echo "-- Install Boost 1.75.0 --" &&
+echo "-- Install Boost 1.86.0 --" &&
 pwd &&
-( wget -nc -q https://boostorg.jfrog.io/artifactory/main/release/1.75.0/source/boost_1_75_0.tar.gz; echo "" ) &&
-tar -xzf boost_1_75_0.tar.gz &&
-cd boost_1_75_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
+( wget -nc -q https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.gz; echo "" ) &&
+tar -xzf boost_1_86_0.tar.gz &&
+cd boost_1_86_0 && ./bootstrap.sh && ./b2  link=static -j$3 && ./b2 install &&
 cd ../ &&
 echo "-- Install Protobuf 3.20.0 --" &&
 pwd &&
@@ -115,7 +122,7 @@ tar -xf libunwind-13.0.1.src.tar.xz &&
 cp -r libunwind-13.0.1.src/include libunwind-13.0.1.src/src lld-13.0.1.src/ &&
 cd lld-13.0.1.src &&
 rm -rf build CMakeCache.txt &&
-mkdir build &&
+mkdir -p build &&
 cd build &&
 cmake .. -DLLVM_LIBRARY_DIR=/usr/lib64/llvm13/lib/ -DCMAKE_INSTALL_PREFIX=/usr/lib64/llvm13/ -DCMAKE_BUILD_TYPE=Release &&
 make -j$3 install &&
@@ -125,11 +132,11 @@ cd ../../ &&
 echo "-- Build WasmEdge --" &&
 ( wget -nc -q https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.11.2.zip; unzip -o 0.11.2.zip; ) &&
 cd WasmEdge-0.11.2 &&
-( mkdir build; echo "" ) &&
+( mkdir -p build; echo "" ) &&
 cd build &&
-export BOOST_ROOT="/usr/local/src/boost_1_75_0" &&
+export BOOST_ROOT="/usr/local/src/boost_1_86_0" &&
 export Boost_LIBRARY_DIRS="/usr/local/lib" &&
-export BOOST_INCLUDEDIR="/usr/local/src/boost_1_75_0" &&
+export BOOST_INCLUDEDIR="/usr/local/src/boost_1_86_0" &&
 export PATH=`echo $PATH | sed -E "s/devtoolset-7/devtoolset-9/g"` &&
 cmake .. \
     -DCMAKE_BUILD_TYPE=Release \

cfg/rippled-example.cfg

@@ -1056,7 +1056,18 @@
 #       Cassandra is an alternative backend to be used only with Reporting Mode.
 #       See the Reporting Mode section for more details about Reporting Mode.
 #
-#   Required keys for NuDB and RocksDB:
+#   type = RWDB
+#
+#       RWDB is a high-performance memory store written by XRPL-Labs and optimized
+#       for xahaud. RWDB is NOT persistent and the data will be lost on restart.
+#       RWDB is recommended for Validator and Peer nodes that are not required to 
+#       store history.
+#
+#       RWDB maintains its high speed regardless of the amount of history
+#       stored. Online delete should NOT be used instead RWDB will use the 
+#       ledger_history config value to determine how many ledgers to keep in memory.
+#
+#   Required keys for NuDB, RWDB and RocksDB:
 #
 #       path                Location to store the database
 #
@@ -1112,7 +1123,8 @@
 #       online_delete       Minimum value of 256. Enable automatic purging
 #                           of older ledger information. Maintain at least this
 #                           number of ledger records online. Must be greater
-#                           than or equal to ledger_history.
+#                           than or equal to ledger_history. If using RWDB 
+#                           this value is ignored.
 #
 #       These keys modify the behavior of online_delete, and thus are only
 #       relevant if online_delete is defined and non-zero:

cfg/rippled-standalone.cfg

@@ -144,4 +144,12 @@ D686F2538F410C9D0D856788E98E3579595DAF7B38D38887F81ECAC934B06040 HooksUpdate1
 86E83A7D2ECE3AD5FA87AB2195AE015C950469ABF0B72EAACED318F74886AE90 CryptoConditionsSuite
 3C43D9A973AA4443EF3FC38E42DD306160FBFFDAB901CD8BAA15D09F2597EB87 NonFungibleTokensV1
 0285B7E5E08E1A8E4C15636F0591D87F73CB6A7B6452A932AD72BBC8E5D1CBE3 fixNFTokenDirV1
-36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+36799EA497B1369B170805C078AEFE6188345F9B3E324C21E9CA3FF574E3C3D6 fixNFTokenNegOffer
+4C499D17719BB365B69010A436B64FD1A82AAB199FC1CEB06962EBD01059FB09 fixXahauV1
+215181D23BF5C173314B5FDB9C872C92DE6CC918483727DE037C0C13E7E6EE9D fixXahauV2
+0D8BF22FF7570D58598D1EF19EBB6E142AD46E59A223FD3816262FBB69345BEA Remit
+7CA0426E7F411D39BB014E57CD9E08F61DE1750F0D41FCD428D9FB80BB7596B0 ZeroB2M
+4B8466415FAB32FFA89D9DCBE166A42340115771DF611A7160F8D7439C87ECD8 fixNSDelete
+EDB4EE4C524E16BDD91D9A529332DED08DCAAA51CC6DC897ACFA1A0ED131C5B6 fix240819
+8063140E9260799D6716756B891CEC3E7006C4E4F277AB84670663A88F94B9C4 fixPageCap
+88693F108C3CD8A967F3F4253A32DEF5E35F9406ACD2A11B88B11D90865763A9 fix240911

hook/generate_sfcodes.sh

@@ -0,0 +1,29 @@
+#/bin/bash
+RIPPLED_ROOT="../src/ripple"
+echo '// For documentation please see: https://xrpl-hooks.readme.io/reference/'
+echo '// Generated using generate_sfcodes.sh'
+cat $RIPPLED_ROOT/protocol/impl/SField.cpp | grep -E '^CONSTRUCT_' |
+    sed 's/UINT16/1/g' |
+    sed 's/UINT32/2/g' |
+    sed 's/UINT64/3/g' |
+    sed 's/HASH128/4/g' |
+    sed 's/HASH256/5/g' |
+    sed 's/UINT128/4/g' |
+    sed 's/UINT256/5/g' |
+    sed 's/AMOUNT/6/g' |
+    sed 's/VL/7/g' | sed 's/Import7/ImportVL/g' |
+    sed 's/ACCOUNT/8/g' |
+    sed 's/OBJECT/14/g' |
+    sed 's/ARRAY/15/g' |
+    sed 's/UINT8/16/g' |
+    sed 's/HASH160/17/g' |
+    sed 's/UINT160/17/g' |
+    sed 's/PATHSET/18/g' |
+    sed 's/VECTOR256/19/g' |
+    sed 's/UINT96/20/g' |
+    sed 's/UINT192/21/g' |
+    sed 's/UINT384/22/g' |
+    sed 's/UINT512/23/g' |
+    grep -Eo '"([^"]+)", *([0-9]+), *([0-9]+)' |
+    sed 's/"//g' | sed 's/ *//g' | sed 's/,/ /g' |
+    awk '{print ("#define sf"$1" (("$2"U << 16U) + "$3"U)")}'

hook/macro.h

@@ -1,5 +1,5 @@
 /**
- * These are helper macros for writing hooks, all of them are optional as is including hookmacro.h at all
+ * These are helper macros for writing hooks, all of them are optional as is including macro.h at all
  */
 
 #include <stdint.h>

hook/sfcodes.h

@@ -60,7 +60,10 @@
 #define sfBurnedNFTokens ((2U << 16U) + 44U)
 #define sfHookStateCount ((2U << 16U) + 45U)
 #define sfEmitGeneration ((2U << 16U) + 46U)
-#define sfLockCount ((2U << 16U) + 47U)
+#define sfLockCount ((2U << 16U) + 49U)
+#define sfFirstNFTokenSequence ((2U << 16U) + 50U)
+#define sfXahauActivationLgrSeq ((2U << 16U) + 96U)
+#define sfImportSequence ((2U << 16U) + 97U)
 #define sfRewardTime ((2U << 16U) + 98U)
 #define sfRewardLgrFirst ((2U << 16U) + 99U)
 #define sfRewardLgrLast ((2U << 16U) + 100U)
@@ -80,6 +83,8 @@
 #define sfHookInstructionCount ((3U << 16U) + 17U)
 #define sfHookReturnCode ((3U << 16U) + 18U)
 #define sfReferenceCount ((3U << 16U) + 19U)
+#define sfAccountIndex ((3U << 16U) + 98U)
+#define sfAccountCount ((3U << 16U) + 99U)
 #define sfRewardAccumulator ((3U << 16U) + 100U)
 #define sfEmailHash ((4U << 16U) + 1U)
 #define sfTakerPaysCurrency ((10U << 16U) + 1U)
@@ -120,6 +125,9 @@
 #define sfOfferID ((5U << 16U) + 34U)
 #define sfEscrowID ((5U << 16U) + 35U)
 #define sfURITokenID ((5U << 16U) + 36U)
+#define sfGovernanceFlags ((5U << 16U) + 99U)
+#define sfGovernanceMarks ((5U << 16U) + 98U)
+#define sfEmittedTxnID ((5U << 16U) + 97U)
 #define sfAmount ((6U << 16U) + 1U)
 #define sfBalance ((6U << 16U) + 2U)
 #define sfLimitAmount ((6U << 16U) + 3U)
@@ -136,6 +144,9 @@
 #define sfNFTokenBrokerFee ((6U << 16U) + 19U)
 #define sfHookCallbackFee ((6U << 16U) + 20U)
 #define sfLockedBalance ((6U << 16U) + 21U)
+#define sfBaseFeeDrops ((6U << 16U) + 22U)
+#define sfReserveBaseDrops ((6U << 16U) + 23U)
+#define sfReserveIncrementDrops ((6U << 16U) + 24U)
 #define sfPublicKey ((7U << 16U) + 1U)
 #define sfMessageKey ((7U << 16U) + 2U)
 #define sfSigningPubKey ((7U << 16U) + 3U)
@@ -171,11 +182,13 @@
 #define sfNFTokenMinter ((8U << 16U) + 9U)
 #define sfEmitCallback ((8U << 16U) + 10U)
 #define sfHookAccount ((8U << 16U) + 16U)
+#define sfInform ((8U << 16U) + 99U)
 #define sfIndexes ((19U << 16U) + 1U)
 #define sfHashes ((19U << 16U) + 2U)
 #define sfAmendments ((19U << 16U) + 3U)
 #define sfNFTokenOffers ((19U << 16U) + 4U)
 #define sfHookNamespaces ((19U << 16U) + 5U)
+#define sfURITokenIDs ((19U << 16U) + 99U)
 #define sfPaths ((18U << 16U) + 1U)
 #define sfTransactionMetaData ((14U << 16U) + 2U)
 #define sfCreatedNode ((14U << 16U) + 3U)
@@ -198,6 +211,12 @@
 #define sfHookDefinition ((14U << 16U) + 22U)
 #define sfHookParameter ((14U << 16U) + 23U)
 #define sfHookGrant ((14U << 16U) + 24U)
+#define sfGenesisMint ((14U << 16U) + 96U)
+#define sfActiveValidator ((14U << 16U) + 95U)
+#define sfImportVLKey ((14U << 16U) + 94U)
+#define sfHookEmission ((14U << 16U) + 93U)
+#define sfMintURIToken ((14U << 16U) + 92U)
+#define sfAmountEntry ((14U << 16U) + 91U)
 #define sfSigners ((15U << 16U) + 3U)
 #define sfSignerEntries ((15U << 16U) + 4U)
 #define sfTemplate ((15U << 16U) + 5U)
@@ -212,4 +231,8 @@
 #define sfHookExecutions ((15U << 16U) + 18U)
 #define sfHookParameters ((15U << 16U) + 19U)
 #define sfHookGrants ((15U << 16U) + 20U)
+#define sfGenesisMints ((15U << 16U) + 96U)
 #define sfActiveValidators ((15U << 16U) + 95U)
+#define sfImportVLKeys ((15U << 16U) + 94U)
+#define sfHookEmissions ((15U << 16U) + 93U)
+#define sfAmounts ((15U << 16U) + 92U)

release-builder.sh

@@ -1,4 +1,11 @@
-#!/bin/bash
+#!/bin/bash -u
+# We use set -e and bash with -u to bail on first non zero exit code of any
+# processes launched or upon any unbound variable.
+# We use set -x to print commands before running them to help with
+# debugging.
+set -ex
+
+set -e
 
 echo "START BUILDING (HOST)"
 
@@ -36,7 +43,8 @@ fi
 
 STATIC_CONTAINER=$(docker ps -a | grep $CONTAINER_NAME |wc -l)
 
-if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then
+#if [[ "$STATIC_CONTAINER" -gt "0" && "$GITHUB_REPOSITORY" != "" ]]; then
+if false; then
   echo "Static container, execute in static container to have max. cache"
   docker start $CONTAINER_NAME
   docker exec -i $CONTAINER_NAME /hbb_exe/activate-exec bash -x /io/build-core.sh "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$BUILD_CORES" "$GITHUB_RUN_NUMBER"

src/ripple/app/consensus/RCLConsensus.cpp

@@ -134,8 +134,12 @@ RCLConsensus::Adaptor::acquireLedger(LedgerHash const& hash)
             acquiringLedger_ = hash;
 
             app_.getJobQueue().addJob(
-                jtADVANCE, "getConsensusLedger", [id = hash, &app = app_]() {
-                    app.getInboundLedgers().acquire(
+                jtADVANCE,
+                "getConsensusLedger1",
+                [id = hash, &app = app_, this]() {
+                    JLOG(j_.debug())
+                        << "JOB advanceLedger getConsensusLedger1 started";
+                    app.getInboundLedgers().acquireAsync(
                         id, 0, InboundLedger::Reason::CONSENSUS);
                 });
         }
@@ -182,7 +186,7 @@ RCLConsensus::Adaptor::share(RCLCxTx const& tx)
     if (app_.getHashRouter().shouldRelay(tx.id()))
     {
         JLOG(j_.debug()) << "Relaying disputed tx " << tx.id();
-        auto const slice = tx.tx_.slice();
+        auto const slice = tx.tx_->slice();
         protocol::TMTransaction msg;
         msg.set_rawtransaction(slice.data(), slice.size());
         msg.set_status(protocol::tsNEW);
@@ -326,7 +330,7 @@ RCLConsensus::Adaptor::onClose(
         tx.first->add(s);
         initialSet->addItem(
             SHAMapNodeType::tnTRANSACTION_NM,
-            SHAMapItem(tx.first->getTransactionID(), s.slice()));
+            make_shamapitem(tx.first->getTransactionID(), s.slice()));
     }
 
     // Add pseudo-transactions to the set
@@ -370,7 +374,8 @@ RCLConsensus::Adaptor::onClose(
         RCLCensorshipDetector<TxID, LedgerIndex>::TxIDSeqVec proposed;
 
         initialSet->visitLeaves(
-            [&proposed, seq](std::shared_ptr<SHAMapItem const> const& item) {
+            [&proposed,
+             seq](boost::intrusive_ptr<SHAMapItem const> const& item) {
                 proposed.emplace_back(item->key(), seq);
             });
 
@@ -535,7 +540,7 @@ RCLConsensus::Adaptor::doAccept(
         std::vector<TxID> accepted;
 
         result.txns.map_->visitLeaves(
-            [&accepted](std::shared_ptr<SHAMapItem const> const& item) {
+            [&accepted](boost::intrusive_ptr<SHAMapItem const> const& item) {
                 accepted.push_back(item->key());
             });
 
@@ -610,7 +615,7 @@ RCLConsensus::Adaptor::doAccept(
                         << "Test applying disputed transaction that did"
                         << " not get in " << dispute.tx().id();
 
-                    SerialIter sit(dispute.tx().tx_.slice());
+                    SerialIter sit(dispute.tx().tx_->slice());
                     auto txn = std::make_shared<STTx const>(sit);
 
                     // Disputed pseudo-transactions that were not accepted

src/ripple/app/consensus/RCLCxTx.h

@@ -42,7 +42,7 @@ public:
 
         @param txn The transaction to wrap
     */
-    RCLCxTx(SHAMapItem const& txn) : tx_{txn}
+    RCLCxTx(boost::intrusive_ptr<SHAMapItem const> txn) : tx_(std::move(txn))
     {
     }
 
@@ -50,11 +50,11 @@ public:
     ID const&
     id() const
     {
-        return tx_.key();
+        return tx_->key();
     }
 
     //! The SHAMapItem that represents the transaction.
-    SHAMapItem const tx_;
+    boost::intrusive_ptr<SHAMapItem const> tx_;
 };
 
 /** Represents a set of transactions in RCLConsensus.
@@ -90,8 +90,7 @@ public:
         bool
         insert(Tx const& t)
         {
-            return map_->addItem(
-                SHAMapNodeType::tnTRANSACTION_NM, SHAMapItem{t.tx_});
+            return map_->addItem(SHAMapNodeType::tnTRANSACTION_NM, t.tx_);
         }
 
         /** Remove a transaction from the set.
@@ -145,7 +144,7 @@ public:
               code uses the shared_ptr semantics to know whether the find
               was successful and properly creates a Tx as needed.
     */
-    std::shared_ptr<const SHAMapItem> const&
+    boost::intrusive_ptr<SHAMapItem const> const&
     find(Tx::ID const& entry) const
     {
         return map_->peekItem(entry);

src/ripple/app/consensus/RCLValidations.cpp

@@ -135,8 +135,10 @@ RCLValidationsAdaptor::acquire(LedgerHash const& hash)
         Application* pApp = &app_;
 
         app_.getJobQueue().addJob(
-            jtADVANCE, "getConsensusLedger", [pApp, hash]() {
-                pApp->getInboundLedgers().acquire(
+            jtADVANCE, "getConsensusLedger2", [pApp, hash, this]() {
+                JLOG(j_.debug())
+                    << "JOB advanceLedger getConsensusLedger2 started";
+                pApp->getInboundLedgers().acquireAsync(
                     hash, 0, InboundLedger::Reason::CONSENSUS);
             });
         return std::nullopt;
@@ -152,7 +154,9 @@ void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,
-    std::string const& source)
+    std::string const& source,
+    BypassAccept const bypassAccept,
+    std::optional<beast::Journal> j)
 {
     auto const& signingKey = val->getSignerPublic();
     auto const& hash = val->getLedgerHash();
@@ -177,7 +181,23 @@ handleNewValidation(
     if (outcome == ValStatus::current)
     {
         if (val->isTrusted())
-            app.getLedgerMaster().checkAccept(hash, seq);
+        {
+            // Was: app.getLedgerMaster().checkAccept(hash, seq);
+            // https://github.com/XRPLF/rippled/commit/fbbea9e6e25795a8a6bd1bf64b780771933a9579
+            if (bypassAccept == BypassAccept::yes)
+            {
+                assert(j.has_value());
+                if (j.has_value())
+                {
+                    JLOG(j->trace()) << "Bypassing checkAccept for validation "
+                                     << val->getLedgerHash();
+                }
+            }
+            else
+            {
+                app.getLedgerMaster().checkAccept(hash, seq);
+            }
+        }
         return;
     }
 

src/ripple/app/consensus/RCLValidations.h

@@ -25,12 +25,16 @@
 #include <ripple/protocol/Protocol.h>
 #include <ripple/protocol/RippleLedgerHash.h>
 #include <ripple/protocol/STValidation.h>
+#include <optional>
+#include <set>
 #include <vector>
 
 namespace ripple {
 
 class Application;
 
+enum class BypassAccept : bool { no = false, yes };
+
 /** Wrapper over STValidation for generic Validation code
 
     Wraps an STValidation for compatibility with the generic validation code.
@@ -248,7 +252,9 @@ void
 handleNewValidation(
     Application& app,
     std::shared_ptr<STValidation> const& val,
-    std::string const& source);
+    std::string const& source,
+    BypassAccept const bypassAccept = BypassAccept::no,
+    std::optional<beast::Journal> j = std::nullopt);
 
 }  // namespace ripple
 

src/ripple/app/hook/Guard.h

@@ -3,6 +3,7 @@
 #include <iostream>
 #include <map>
 #include <memory>
+#include <optional>
 #include <ostream>
 #include <stack>
 #include <string>
@@ -271,7 +272,19 @@ check_guard(
     int guard_func_idx,
     int last_import_idx,
     GuardLog guardLog,
-    std::string guardLogAccStr)
+    std::string guardLogAccStr,
+    /* RH NOTE:
+     * rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
+     * and update 3 is 0x04 ideally at rule version 3 all bits so far are set
+     * (0b111) so the ruleVersion = 7, however if a specific rule update must be
+     * rolled back due to unforeseen behaviour then this may no longer be the
+     * case. using a bit field here leaves us flexible to rollback changes that
+     * might have unforeseen consequences, without also rolling back further
+     * changes that are fine.
+     */
+    uint64_t rulesVersion = 0
+
+)
 {
 #define MAX_GUARD_CALLS 1024
     uint32_t guard_count = 0;
@@ -621,11 +634,17 @@ check_guard(
             }
             else if (fc_type == 10)  // memory.copy
             {
+                if (rulesVersion & 0x02U)
+                    GUARD_ERROR("Memory.copy instruction is not allowed.");
+
                 REQUIRE(2);
                 ADVANCE(2);
             }
             else if (fc_type == 11)  // memory.fill
             {
+                if (rulesVersion & 0x02U)
+                    GUARD_ERROR("Memory.fill instruction is not allowed.");
+
                 ADVANCE(1);
             }
             else if (fc_type <= 7)  // numeric instructions
@@ -807,6 +826,15 @@ validateGuards(
     std::vector<uint8_t> const& wasm,
     GuardLog guardLog,
     std::string guardLogAccStr,
+    /* RH NOTE:
+     * rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
+     * and update 3 is 0x04 ideally at rule version 3 all bits so far are set
+     * (0b111) so the ruleVersion = 7, however if a specific rule update must be
+     * rolled back due to unforeseen behaviour then this may no longer be the
+     * case. using a bit field here leaves us flexible to rollback changes that
+     * might have unforeseen consequences, without also rolling back further
+     * changes that are fine.
+     */
     uint64_t rulesVersion = 0)
 {
     uint64_t byteCount = wasm.size();
@@ -1477,7 +1505,8 @@ validateGuards(
                     guard_import_number,
                     last_import_number,
                     guardLog,
-                    guardLogAccStr);
+                    guardLogAccStr,
+                    rulesVersion);
 
                 if (!valid)
                     return {};

src/ripple/app/hook/guard_checker.cpp

@@ -79,7 +79,7 @@ main(int argc, char** argv)
 
     close(fd);
 
-    auto result = validateGuards(hook, std::cout, "", 1);
+    auto result = validateGuards(hook, std::cout, "", 3);
 
     if (!result)
     {

src/ripple/app/hook/impl/applyHook.cpp

@@ -1,12 +1,17 @@
 #include <ripple/app/hook/applyHook.h>
 #include <ripple/app/ledger/OpenLedger.h>
 #include <ripple/app/ledger/TransactionMaster.h>
+#include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/NetworkOPs.h>
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/TxQ.h>
+#include <ripple/app/tx/impl/Import.h>
+#include <ripple/app/tx/impl/details/NFTokenUtils.h>
 #include <ripple/basics/Log.h>
 #include <ripple/basics/Slice.h>
 #include <ripple/protocol/ErrorCodes.h>
+#include <ripple/protocol/TxFlags.h>
+#include <ripple/protocol/st.h>
 #include <ripple/protocol/tokens.h>
 #include <boost/multiprecision/cpp_dec_float.hpp>
 #include <any>
@@ -1212,9 +1217,10 @@ hook::apply(
              .hookParamOverrides = hookParamOverrides,
              .hookParams = hookParams,
              .hookSkips = {},
-             .exitType =
-                 hook_api::ExitType::ROLLBACK,  // default is to rollback unless
-                                                // hook calls accept()
+             .exitType = applyCtx.view().rules().enabled(fixXahauV3)
+                 ? hook_api::ExitType::UNSET
+                 : hook_api::ExitType::ROLLBACK,  // default is to rollback
+                                                  // unless hook calls accept()
              .exitReason = std::string(""),
              .exitCode = -1,
              .hasCallback = hasCallback,
@@ -4612,6 +4618,8 @@ DEFINE_HOOK_FUNCTION(
     }
     catch (std::exception& e)
     {
+        JLOG(j.trace()) << "HookInfo[" << HC_ACC()
+                        << "]: etxn_fee_base exception: " << e.what();
         return INVALID_TXN;
     }
 
@@ -4783,7 +4791,7 @@ DEFINE_HOOK_FUNCTION(
 
     if (float1 == 0)
     {
-        j.trace() << "HookTrace[" << HC_ACC() << "]:"
+        j.trace() << "HookTrace[" << HC_ACC() << "]: "
                   << (read_len == 0
                           ? ""
                           : std::string_view(
@@ -5397,7 +5405,7 @@ DEFINE_HOOK_FUNCTION(
 const int64_t float_one_internal = make_float(1000000000000000ull, -15, false);
 
 inline int64_t
-float_divide_internal(int64_t float1, int64_t float2)
+float_divide_internal(int64_t float1, int64_t float2, bool hasFix)
 {
     RETURN_IF_INVALID_FLOAT(float1);
     RETURN_IF_INVALID_FLOAT(float2);
@@ -5450,8 +5458,16 @@ float_divide_internal(int64_t float1, int64_t float2)
     while (man2 > 0)
     {
         int i = 0;
-        for (; man1 > man2; man1 -= man2, ++i)
-            ;
+        if (hasFix)
+        {
+            for (; man1 >= man2; man1 -= man2, ++i)
+                ;
+        }
+        else
+        {
+            for (; man1 > man2; man1 -= man2, ++i)
+                ;
+        }
 
         man3 *= 10;
         man3 += i;
@@ -5471,7 +5487,8 @@ DEFINE_HOOK_FUNCTION(int64_t, float_divide, int64_t float1, int64_t float2)
     HOOK_SETUP();  // populates memory_ctx, memory, memory_length, applyCtx,
                    // hookCtx on current stack
 
-    return float_divide_internal(float1, float2);
+    bool const hasFix = view.rules().enabled(fixFloatDivide);
+    return float_divide_internal(float1, float2, hasFix);
 
     HOOK_TEARDOWN();
 }
@@ -5490,7 +5507,9 @@ DEFINE_HOOK_FUNCTION(int64_t, float_invert, int64_t float1)
         return DIVISION_BY_ZERO;
     if (float1 == float_one_internal)
         return float_one_internal;
-    return float_divide_internal(float_one_internal, float1);
+
+    bool const fixV3 = view.rules().enabled(fixFloatDivide);
+    return float_divide_internal(float_one_internal, float1, fixV3);
 
     HOOK_TEARDOWN();
 }

src/ripple/app/ledger/InboundLedgers.h

@@ -38,10 +38,21 @@ public:
     virtual ~InboundLedgers() = default;
 
     // VFALCO TODO Should this be called findOrAdd ?
+    // Callers should use this if they possibly need an authoritative
+    // response immediately.
     //
     virtual std::shared_ptr<Ledger const>
     acquire(uint256 const& hash, std::uint32_t seq, InboundLedger::Reason) = 0;
 
+    // Callers should use this if they are known to be executing on the Job
+    // Queue. TODO review whether all callers of acquire() can use this
+    // instead. Inbound ledger acquisition is asynchronous anyway.
+    virtual void
+    acquireAsync(
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) = 0;
+
     virtual std::shared_ptr<InboundLedger>
     find(LedgerHash const& hash) = 0;
 

src/ripple/app/ledger/Ledger.cpp

@@ -119,9 +119,8 @@ public:
     sles_type::value_type
     dereference() const override
     {
-        auto const item = *iter_;
-        SerialIter sit(item.slice());
-        return std::make_shared<SLE const>(sit, item.key());
+        SerialIter sit(iter_->slice());
+        return std::make_shared<SLE const>(sit, iter_->key());
     }
 };
 
@@ -168,7 +167,7 @@ public:
     txs_type::value_type
     dereference() const override
     {
-        auto const item = *iter_;
+        auto const& item = *iter_;
         if (metadata_)
             return deserializeTxPlusMeta(item);
         return {deserializeTx(item), nullptr};
@@ -183,8 +182,8 @@ Ledger::Ledger(
     std::vector<uint256> const& amendments,
     Family& family)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(SHAMapType::TRANSACTION, family))
-    , stateMap_(std::make_shared<SHAMap>(SHAMapType::STATE, family))
+    , txMap_(SHAMapType::TRANSACTION, family)
+    , stateMap_(SHAMapType::STATE, family)
     , rules_{config.features}
     , j_(beast::Journal(beast::Journal::getNullSink()))
 {
@@ -247,7 +246,7 @@ Ledger::Ledger(
         rawInsert(sle);
     }
 
-    stateMap_->flushDirty(hotACCOUNT_NODE);
+    stateMap_.flushDirty(hotACCOUNT_NODE);
     setImmutable();
 }
 
@@ -259,12 +258,8 @@ Ledger::Ledger(
     Family& family,
     beast::Journal j)
     : mImmutable(true)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          info.txHash,
-          family))
-    , stateMap_(
-          std::make_shared<SHAMap>(SHAMapType::STATE, info.accountHash, family))
+    , txMap_(SHAMapType::TRANSACTION, info.txHash, family)
+    , stateMap_(SHAMapType::STATE, info.accountHash, family)
     , rules_(config.features)
     , info_(info)
     , j_(j)
@@ -272,7 +267,7 @@ Ledger::Ledger(
     loaded = true;
 
     if (info_.txHash.isNonZero() &&
-        !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
+        !txMap_.fetchRoot(SHAMapHash{info_.txHash}, nullptr))
     {
         if (config.reporting())
         {
@@ -284,7 +279,7 @@ Ledger::Ledger(
     }
 
     if (info_.accountHash.isNonZero() &&
-        !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
+        !stateMap_.fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
     {
         if (config.reporting())
         {
@@ -295,8 +290,8 @@ Ledger::Ledger(
         JLOG(j.warn()) << "Don't have state data root for ledger" << info_.seq;
     }
 
-    txMap_->setImmutable();
-    stateMap_->setImmutable();
+    txMap_.setImmutable();
+    stateMap_.setImmutable();
 
     defaultFees(config);
     if (!setup())
@@ -313,10 +308,8 @@ Ledger::Ledger(
 // Create a new ledger that follows this one
 Ledger::Ledger(Ledger const& prevLedger, NetClock::time_point closeTime)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          prevLedger.stateMap_->family()))
-    , stateMap_(prevLedger.stateMap_->snapShot(true))
+    , txMap_(SHAMapType::TRANSACTION, prevLedger.txMap_.family())
+    , stateMap_(prevLedger.stateMap_, true)
     , fees_(prevLedger.fees_)
     , rules_(prevLedger.rules_)
     , j_(beast::Journal(beast::Journal::getNullSink()))
@@ -345,12 +338,8 @@ Ledger::Ledger(Ledger const& prevLedger, NetClock::time_point closeTime)
 
 Ledger::Ledger(LedgerInfo const& info, Config const& config, Family& family)
     : mImmutable(true)
-    , txMap_(std::make_shared<SHAMap>(
-          SHAMapType::TRANSACTION,
-          info.txHash,
-          family))
-    , stateMap_(
-          std::make_shared<SHAMap>(SHAMapType::STATE, info.accountHash, family))
+    , txMap_(SHAMapType::TRANSACTION, info.txHash, family)
+    , stateMap_(SHAMapType::STATE, info.accountHash, family)
     , rules_{config.features}
     , info_(info)
     , j_(beast::Journal(beast::Journal::getNullSink()))
@@ -364,8 +353,8 @@ Ledger::Ledger(
     Config const& config,
     Family& family)
     : mImmutable(false)
-    , txMap_(std::make_shared<SHAMap>(SHAMapType::TRANSACTION, family))
-    , stateMap_(std::make_shared<SHAMap>(SHAMapType::STATE, family))
+    , txMap_(SHAMapType::TRANSACTION, family)
+    , stateMap_(SHAMapType::STATE, family)
     , rules_{config.features}
     , j_(beast::Journal(beast::Journal::getNullSink()))
 {
@@ -383,16 +372,16 @@ Ledger::setImmutable(bool rehash)
     // place the hash transitions to valid
     if (!mImmutable && rehash)
     {
-        info_.txHash = txMap_->getHash().as_uint256();
-        info_.accountHash = stateMap_->getHash().as_uint256();
+        info_.txHash = txMap_.getHash().as_uint256();
+        info_.accountHash = stateMap_.getHash().as_uint256();
     }
 
     if (rehash)
         info_.hash = calculateLedgerHash(info_);
 
     mImmutable = true;
-    txMap_->setImmutable();
-    stateMap_->setImmutable();
+    txMap_.setImmutable();
+    stateMap_.setImmutable();
     setup();
 }
 
@@ -415,8 +404,8 @@ bool
 Ledger::addSLE(SLE const& sle)
 {
     auto const s = sle.getSerializer();
-    SHAMapItem item(sle.key(), s.slice());
-    return stateMap_->addItem(SHAMapNodeType::tnACCOUNT_STATE, std::move(item));
+    return stateMap_.addItem(
+        SHAMapNodeType::tnACCOUNT_STATE, make_shamapitem(sle.key(), s.slice()));
 }
 
 //------------------------------------------------------------------------------
@@ -451,20 +440,20 @@ bool
 Ledger::exists(Keylet const& k) const
 {
     // VFALCO NOTE Perhaps check the type for debug builds?
-    return stateMap_->hasItem(k.key);
+    return stateMap_.hasItem(k.key);
 }
 
 bool
 Ledger::exists(uint256 const& key) const
 {
-    return stateMap_->hasItem(key);
+    return stateMap_.hasItem(key);
 }
 
 std::optional<uint256>
 Ledger::succ(uint256 const& key, std::optional<uint256> const& last) const
 {
-    auto item = stateMap_->upper_bound(key);
-    if (item == stateMap_->end())
+    auto item = stateMap_.upper_bound(key);
+    if (item == stateMap_.end())
         return std::nullopt;
     if (last && item->key() >= last)
         return std::nullopt;
@@ -479,7 +468,7 @@ Ledger::read(Keylet const& k) const
         assert(false);
         return nullptr;
     }
-    auto const& item = stateMap_->peekItem(k.key);
+    auto const& item = stateMap_.peekItem(k.key);
     if (!item)
         return nullptr;
     auto sle = std::make_shared<SLE>(SerialIter{item->slice()}, item->key());
@@ -493,45 +482,44 @@ Ledger::read(Keylet const& k) const
 auto
 Ledger::slesBegin() const -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->begin());
+    return std::make_unique<sles_iter_impl>(stateMap_.begin());
 }
 
 auto
 Ledger::slesEnd() const -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->end());
+    return std::make_unique<sles_iter_impl>(stateMap_.end());
 }
 
 auto
 Ledger::slesUpperBound(uint256 const& key) const
     -> std::unique_ptr<sles_type::iter_base>
 {
-    return std::make_unique<sles_iter_impl>(stateMap_->upper_bound(key));
+    return std::make_unique<sles_iter_impl>(stateMap_.upper_bound(key));
 }
 
 auto
 Ledger::txsBegin() const -> std::unique_ptr<txs_type::iter_base>
 {
-    return std::make_unique<txs_iter_impl>(!open(), txMap_->begin());
+    return std::make_unique<txs_iter_impl>(!open(), txMap_.begin());
 }
 
 auto
 Ledger::txsEnd() const -> std::unique_ptr<txs_type::iter_base>
 {
-    return std::make_unique<txs_iter_impl>(!open(), txMap_->end());
+    return std::make_unique<txs_iter_impl>(!open(), txMap_.end());
 }
 
 bool
 Ledger::txExists(uint256 const& key) const
 {
-    return txMap_->hasItem(key);
+    return txMap_.hasItem(key);
 }
 
 auto
 Ledger::txRead(key_type const& key) const -> tx_type
 {
-    assert(txMap_);
-    auto const& item = txMap_->peekItem(key);
+    auto const& item = txMap_.peekItem(key);
     if (!item)
         return {};
     if (!open())
@@ -548,7 +536,7 @@ Ledger::digest(key_type const& key) const -> std::optional<digest_type>
     SHAMapHash digest;
     // VFALCO Unfortunately this loads the item
     //        from the NodeStore needlessly.
-    if (!stateMap_->peekItem(key, digest))
+    if (!stateMap_.peekItem(key, digest))
         return std::nullopt;
     return digest.as_uint256();
 }
@@ -558,14 +546,14 @@ Ledger::digest(key_type const& key) const -> std::optional<digest_type>
 void
 Ledger::rawErase(std::shared_ptr<SLE> const& sle)
 {
-    if (!stateMap_->delItem(sle->key()))
+    if (!stateMap_.delItem(sle->key()))
         LogicError("Ledger::rawErase: key not found");
 }
 
 void
 Ledger::rawErase(uint256 const& key)
 {
-    if (!stateMap_->delItem(key))
+    if (!stateMap_.delItem(key))
         LogicError("Ledger::rawErase: key not found");
 }
 
@@ -574,9 +562,9 @@ Ledger::rawInsert(std::shared_ptr<SLE> const& sle)
 {
     Serializer ss;
     sle->add(ss);
-    if (!stateMap_->addGiveItem(
+    if (!stateMap_.addGiveItem(
             SHAMapNodeType::tnACCOUNT_STATE,
-            std::make_shared<SHAMapItem const>(sle->key(), ss.slice())))
+            make_shamapitem(sle->key(), ss.slice())))
         LogicError("Ledger::rawInsert: key already exists");
 }
 
@@ -585,9 +573,9 @@ Ledger::rawReplace(std::shared_ptr<SLE> const& sle)
 {
     Serializer ss;
     sle->add(ss);
-    if (!stateMap_->updateGiveItem(
+    if (!stateMap_.updateGiveItem(
             SHAMapNodeType::tnACCOUNT_STATE,
-            std::make_shared<SHAMapItem const>(sle->key(), ss.slice())))
+            make_shamapitem(sle->key(), ss.slice())))
         LogicError("Ledger::rawReplace: key not found");
 }
 
@@ -603,9 +591,8 @@ Ledger::rawTxInsert(
     Serializer s(txn->getDataLength() + metaData->getDataLength() + 16);
     s.addVL(txn->peekData());
     s.addVL(metaData->peekData());
-    if (!txMap().addGiveItem(
-            SHAMapNodeType::tnTRANSACTION_MD,
-            std::make_shared<SHAMapItem const>(key, s.slice())))
+    if (!txMap_.addGiveItem(
+            SHAMapNodeType::tnTRANSACTION_MD, make_shamapitem(key, s.slice())))
         LogicError("duplicate_tx: " + to_string(key));
 }
 
@@ -621,9 +608,9 @@ Ledger::rawTxInsertWithHash(
     Serializer s(txn->getDataLength() + metaData->getDataLength() + 16);
     s.addVL(txn->peekData());
     s.addVL(metaData->peekData());
-    auto item = std::make_shared<SHAMapItem const>(key, s.slice());
+    auto item = make_shamapitem(key, s.slice());
     auto hash = sha512Half(HashPrefix::txNode, item->slice(), item->key());
-    if (!txMap().addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, std::move(item)))
+    if (!txMap_.addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, std::move(item)))
         LogicError("duplicate_tx: " + to_string(key));
 
     return hash;
@@ -723,7 +710,7 @@ Ledger::defaultFees(Config const& config)
 std::shared_ptr<SLE>
 Ledger::peek(Keylet const& k) const
 {
-    auto const& value = stateMap_->peekItem(k.key);
+    auto const& value = stateMap_.peekItem(k.key);
     if (!value)
         return nullptr;
     auto sle = std::make_shared<SLE>(SerialIter{value->slice()}, value->key());
@@ -845,8 +832,8 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
     std::vector<SHAMapMissingNode> missingNodes1;
     std::vector<SHAMapMissingNode> missingNodes2;
 
-    if (stateMap_->getHash().isZero() && !info_.accountHash.isZero() &&
-        !stateMap_->fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
+    if (stateMap_.getHash().isZero() && !info_.accountHash.isZero() &&
+        !stateMap_.fetchRoot(SHAMapHash{info_.accountHash}, nullptr))
     {
         missingNodes1.emplace_back(
             SHAMapType::STATE, SHAMapHash{info_.accountHash});
@@ -854,9 +841,9 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
     else
     {
         if (parallel)
-            return stateMap_->walkMapParallel(missingNodes1, 32);
+            return stateMap_.walkMapParallel(missingNodes1, 32);
         else
-            stateMap_->walkMap(missingNodes1, 32);
+            stateMap_.walkMap(missingNodes1, 32);
     }
 
     if (!missingNodes1.empty())
@@ -868,15 +855,15 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
         }
     }
 
-    if (txMap_->getHash().isZero() && info_.txHash.isNonZero() &&
-        !txMap_->fetchRoot(SHAMapHash{info_.txHash}, nullptr))
+    if (txMap_.getHash().isZero() && info_.txHash.isNonZero() &&
+        !txMap_.fetchRoot(SHAMapHash{info_.txHash}, nullptr))
     {
         missingNodes2.emplace_back(
             SHAMapType::TRANSACTION, SHAMapHash{info_.txHash});
     }
     else
     {
-        txMap_->walkMap(missingNodes2, 32);
+        txMap_.walkMap(missingNodes2, 32);
     }
 
     if (!missingNodes2.empty())
@@ -893,9 +880,9 @@ Ledger::walkLedger(beast::Journal j, bool parallel) const
 bool
 Ledger::assertSensible(beast::Journal ledgerJ) const
 {
-    if (info_.hash.isNonZero() && info_.accountHash.isNonZero() && stateMap_ &&
-        txMap_ && (info_.accountHash == stateMap_->getHash().as_uint256()) &&
-        (info_.txHash == txMap_->getHash().as_uint256()))
+    if (info_.hash.isNonZero() && info_.accountHash.isNonZero() &&
+        (info_.accountHash == stateMap_.getHash().as_uint256()) &&
+        (info_.txHash == txMap_.getHash().as_uint256()))
     {
         return true;
     }
@@ -1057,15 +1044,14 @@ pendSaveValidated(
         return true;
     }
 
-    JobType const jobType{isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER};
-    char const* const jobName{
-        isCurrent ? "Ledger::pendSave" : "Ledger::pendOldSave"};
-
     // See if we can use the JobQueue.
     if (!isSynchronous &&
-        app.getJobQueue().addJob(jobType, jobName, [&app, ledger, isCurrent]() {
-            saveValidatedLedger(app, ledger, isCurrent);
-        }))
+        app.getJobQueue().addJob(
+            isCurrent ? jtPUBLEDGER : jtPUBOLDLEDGER,
+            std::to_string(ledger->seq()),
+            [&app, ledger, isCurrent]() {
+                saveValidatedLedger(app, ledger, isCurrent);
+            }))
     {
         return true;
     }
@@ -1077,15 +1063,15 @@ pendSaveValidated(
 void
 Ledger::unshare() const
 {
-    stateMap_->unshare();
-    txMap_->unshare();
+    stateMap_.unshare();
+    txMap_.unshare();
 }
 
 void
 Ledger::invariants() const
 {
-    stateMap_->invariants();
-    txMap_->invariants();
+    stateMap_.invariants();
+    txMap_.invariants();
 }
 //------------------------------------------------------------------------------
 

src/ripple/app/ledger/Ledger.h

@@ -83,6 +83,10 @@ public:
     Ledger&
     operator=(Ledger const&) = delete;
 
+    Ledger(Ledger&&) = delete;
+    Ledger&
+    operator=(Ledger&&) = delete;
+
     /** Create the Genesis ledger.
 
         The Genesis ledger contains a single account whose
@@ -290,10 +294,10 @@ public:
     void
     setFull() const
     {
-        txMap_->setFull();
-        stateMap_->setFull();
-        txMap_->setLedgerSeq(info_.seq);
-        stateMap_->setLedgerSeq(info_.seq);
+        txMap_.setFull();
+        txMap_.setLedgerSeq(info_.seq);
+        stateMap_.setFull();
+        stateMap_.setLedgerSeq(info_.seq);
     }
 
     void
@@ -305,25 +309,25 @@ public:
     SHAMap const&
     stateMap() const
     {
-        return *stateMap_;
+        return stateMap_;
     }
 
     SHAMap&
     stateMap()
     {
-        return *stateMap_;
+        return stateMap_;
     }
 
     SHAMap const&
     txMap() const
     {
-        return *txMap_;
+        return txMap_;
     }
 
     SHAMap&
     txMap()
     {
-        return *txMap_;
+        return txMap_;
     }
 
     // returns false on error
@@ -401,8 +405,11 @@ private:
 
     bool mImmutable;
 
-    std::shared_ptr<SHAMap> txMap_;
-    std::shared_ptr<SHAMap> stateMap_;
+    // A SHAMap containing the transactions associated with this ledger.
+    SHAMap mutable txMap_;
+
+    // A SHAMap containing the state objects for this ledger.
+    SHAMap mutable stateMap_;
 
     // Protects fee variables
     std::mutex mutable mutex_;

src/ripple/app/ledger/LedgerHistory.cpp

@@ -51,7 +51,9 @@ LedgerHistory::LedgerHistory(
 }
 
 bool
-LedgerHistory::insert(std::shared_ptr<Ledger const> ledger, bool validated)
+LedgerHistory::insert(
+    std::shared_ptr<Ledger const> const& ledger,
+    bool validated)
 {
     if (!ledger->isImmutable())
         LogicError("mutable Ledger in insert");
@@ -72,12 +74,9 @@ LedgerHash
 LedgerHistory::getLedgerHash(LedgerIndex index)
 {
     std::unique_lock sl(m_ledgers_by_hash.peekMutex());
-    auto it = mLedgersByIndex.find(index);
-
-    if (it != mLedgersByIndex.end())
+    if (auto it = mLedgersByIndex.find(index); it != mLedgersByIndex.end())
         return it->second;
-
-    return uint256();
+    return {};
 }
 
 std::shared_ptr<Ledger const>
@@ -167,19 +166,19 @@ log_metadata_difference(
     uint256 const& tx,
     beast::Journal j)
 {
-    auto getMeta = [](ReadView const& ledger,
-                      uint256 const& txID) -> std::shared_ptr<TxMeta> {
-        auto meta = ledger.txRead(txID).second;
-        if (!meta)
-            return {};
-        return std::make_shared<TxMeta>(txID, ledger.seq(), *meta);
+    auto getMeta = [](ReadView const& ledger, uint256 const& txID) {
+        std::optional<TxMeta> ret;
+        if (auto meta = ledger.txRead(txID).second)
+            ret.emplace(txID, ledger.seq(), *meta);
+        return ret;
     };
 
     auto validMetaData = getMeta(validLedger, tx);
     auto builtMetaData = getMeta(builtLedger, tx);
-    assert(validMetaData != nullptr || builtMetaData != nullptr);
 
-    if (validMetaData != nullptr && builtMetaData != nullptr)
+    assert(validMetaData || builtMetaData);
+
+    if (validMetaData && builtMetaData)
     {
         auto const& validNodes = validMetaData->getNodes();
         auto const& builtNodes = builtMetaData->getNodes();
@@ -280,17 +279,21 @@ log_metadata_difference(
                                 << validNodes.getJson(JsonOptions::none);
             }
         }
+
+        return;
     }
-    else if (validMetaData != nullptr)
+
+    if (validMetaData)
     {
         JLOG(j.error()) << "MISMATCH on TX " << tx
-                        << ": Metadata Difference (built has none)\n"
+                        << ": Metadata Difference. Valid=\n"
                         << validMetaData->getJson(JsonOptions::none);
     }
-    else  // builtMetaData != nullptr
+
+    if (builtMetaData)
     {
         JLOG(j.error()) << "MISMATCH on TX " << tx
-                        << ": Metadata Difference (valid has none)\n"
+                        << ": Metadata Difference. Built=\n"
                         << builtMetaData->getJson(JsonOptions::none);
     }
 }

src/ripple/app/ledger/LedgerHistory.h

@@ -44,7 +44,7 @@ public:
         @return `true` if the ledger was already tracked
     */
     bool
-    insert(std::shared_ptr<Ledger const> ledger, bool validated);
+    insert(std::shared_ptr<Ledger const> const& ledger, bool validated);
 
     /** Get the ledgers_by_hash cache hit rate
         @return the hit rate

src/ripple/app/ledger/LedgerMaster.h

@@ -152,6 +152,9 @@ public:
     std::string
     getCompleteLedgers();
 
+    RangeSet<std::uint32_t>
+    getCompleteLedgersRangeSet();
+
     /** Apply held transactions to the open ledger
         This is normally called as we close the ledger.
         The open ledger remains open to handle new transactions

src/ripple/app/ledger/LedgerReplayer.h

@@ -105,7 +105,7 @@ public:
     void
     gotSkipList(
         LedgerInfo const& info,
-        std::shared_ptr<SHAMapItem const> const& data);
+        boost::intrusive_ptr<SHAMapItem const> const& data);
 
     /**
      * Process a ledger delta (extracted from a TMReplayDeltaResponse message)

src/ripple/app/ledger/TransactionMaster.h

@@ -68,7 +68,7 @@ public:
 
     std::shared_ptr<STTx const>
     fetch(
-        std::shared_ptr<SHAMapItem> const& item,
+        boost::intrusive_ptr<SHAMapItem> const& item,
         SHAMapNodeType type,
         std::uint32_t uCommitLedger);
 

src/ripple/app/ledger/impl/InboundLedger.cpp

@@ -560,7 +560,7 @@ InboundLedger::trigger(std::shared_ptr<Peer> const& peer, TriggerReason reason)
         return;
     }
 
-    if (auto stream = journal_.trace())
+    if (auto stream = journal_.debug())
     {
         if (peer)
             stream << "Trigger acquiring ledger " << hash_ << " from " << peer;

src/ripple/app/ledger/impl/InboundLedgers.cpp

@@ -28,6 +28,7 @@
 #include <ripple/core/JobQueue.h>
 #include <ripple/nodestore/DatabaseShard.h>
 #include <ripple/protocol/jss.h>
+#include <exception>
 #include <memory>
 #include <mutex>
 #include <vector>
@@ -141,6 +142,37 @@ public:
         return inbound->getLedger();
     }
 
+    void
+    acquireAsync(
+        uint256 const& hash,
+        std::uint32_t seq,
+        InboundLedger::Reason reason) override
+    {
+        std::unique_lock lock(acquiresMutex_);
+        try
+        {
+            if (pendingAcquires_.contains(hash))
+                return;
+            pendingAcquires_.insert(hash);
+            lock.unlock();
+            acquire(hash, seq, reason);
+        }
+        catch (std::exception const& e)
+        {
+            JLOG(j_.warn())
+                << "Exception thrown for acquiring new inbound ledger " << hash
+                << ": " << e.what();
+        }
+        catch (...)
+        {
+            JLOG(j_.warn())
+                << "Unknown exception thrown for acquiring new inbound ledger "
+                << hash;
+        }
+        lock.lock();
+        pendingAcquires_.erase(hash);
+    }
+
     std::shared_ptr<InboundLedger>
     find(uint256 const& hash) override
     {
@@ -426,6 +458,9 @@ private:
     beast::insight::Counter mCounter;
 
     std::unique_ptr<PeerSetBuilder> mPeerSetBuilder;
+
+    std::set<uint256> pendingAcquires_;
+    std::mutex acquiresMutex_;
 };
 
 //------------------------------------------------------------------------------

src/ripple/app/ledger/impl/LedgerCleaner.cpp

@@ -219,7 +219,7 @@ private:
     run()
     {
         beast::setCurrentThreadName("LedgerCleaner");
-        JLOG(j_.debug()) << "Started";
+        JLOG(j_.debug()) << "Started ledger cleaner";
 
         while (true)
         {
@@ -392,7 +392,8 @@ private:
 
             if (app_.getFeeTrack().isLoadedLocal())
             {
-                JLOG(j_.debug()) << "Waiting for load to subside";
+                JLOG(j_.debug())
+                    << "Ledger Cleaner: Waiting for load to subside";
                 std::this_thread::sleep_for(std::chrono::seconds(5));
                 continue;
             }
@@ -415,13 +416,15 @@ private:
             bool fail = false;
             if (ledgerHash.isZero())
             {
-                JLOG(j_.info())
-                    << "Unable to get hash for ledger " << ledgerIndex;
+                JLOG(j_.warn())
+                    << "Ledger Cleaner: Unable to get hash for ledger "
+                    << ledgerIndex;
                 fail = true;
             }
             else if (!doLedger(ledgerIndex, ledgerHash, doNodes, doTxns))
             {
-                JLOG(j_.info()) << "Failed to process ledger " << ledgerIndex;
+                JLOG(j_.warn()) << "Ledger Cleaner: Failed to process ledger "
+                                << ledgerIndex;
                 fail = true;
             }
 

src/ripple/app/ledger/impl/LedgerMaster.cpp

@@ -1714,6 +1714,13 @@ LedgerMaster::getCompleteLedgers()
     return to_string(mCompleteLedgers);
 }
 
+RangeSet<std::uint32_t>
+LedgerMaster::getCompleteLedgersRangeSet()
+{
+    std::lock_guard sl(mCompleteLock);
+    return mCompleteLedgers;
+}
+
 std::optional<NetClock::time_point>
 LedgerMaster::getCloseTimeBySeq(LedgerIndex ledgerIndex)
 {

src/ripple/app/ledger/impl/LedgerReplayMsgHandler.cpp

@@ -163,15 +163,15 @@ LedgerReplayMsgHandler::processProofPathResponse(
         JLOG(journal_.debug()) << "Bad message: Cannot deserialize";
         return false;
     }
-    auto item = static_cast<SHAMapLeafNode*>(node.get())->peekItem();
-    if (!item)
+
+    if (auto item = static_cast<SHAMapLeafNode*>(node.get())->peekItem())
     {
-        JLOG(journal_.debug()) << "Bad message: Cannot get ShaMapItem";
-        return false;
+        replayer_.gotSkipList(info, item);
+        return true;
     }
 
-    replayer_.gotSkipList(info, item);
-    return true;
+    JLOG(journal_.debug()) << "Bad message: Cannot get ShaMapItem";
+    return false;
 }
 
 protocol::TMReplayDeltaResponse
@@ -206,9 +206,10 @@ LedgerReplayMsgHandler::processReplayDeltaRequest(
     reply.set_ledgerheader(nData.getDataPtr(), nData.getLength());
     // pack transactions
     auto const& txMap = ledger->txMap();
-    txMap.visitLeaves([&](std::shared_ptr<SHAMapItem const> const& txNode) {
-        reply.add_transaction(txNode->data(), txNode->size());
-    });
+    txMap.visitLeaves(
+        [&](boost::intrusive_ptr<SHAMapItem const> const& txNode) {
+            reply.add_transaction(txNode->data(), txNode->size());
+        });
 
     JLOG(journal_.debug()) << "getReplayDelta for ledger " << ledgerHash
                            << " txMap hash " << txMap.getHash().as_uint256();
@@ -264,10 +265,9 @@ LedgerReplayMsgHandler::processReplayDeltaResponse(
             STObject meta(metaSit, sfMetadata);
             orderedTxns.emplace(meta[sfTransactionIndex], std::move(tx));
 
-            auto item =
-                std::make_shared<SHAMapItem const>(tid, shaMapItemData.slice());
-            if (!item ||
-                !txMap.addGiveItem(SHAMapNodeType::tnTRANSACTION_MD, item))
+            if (!txMap.addGiveItem(
+                    SHAMapNodeType::tnTRANSACTION_MD,
+                    make_shamapitem(tid, shaMapItemData.slice())))
             {
                 JLOG(journal_.debug()) << "Bad message: Cannot deserialize";
                 return false;

src/ripple/app/ledger/impl/LedgerReplayer.cpp

@@ -172,7 +172,7 @@ LedgerReplayer::createDeltas(std::shared_ptr<LedgerReplayTask> task)
 void
 LedgerReplayer::gotSkipList(
     LedgerInfo const& info,
-    std::shared_ptr<SHAMapItem const> const& item)
+    boost::intrusive_ptr<SHAMapItem const> const& item)
 {
     std::shared_ptr<SkipListAcquire> skipList = {};
     {

src/ripple/app/ledger/impl/LedgerToJson.cpp

@@ -131,14 +131,14 @@ fillJsonTx(
         if (stMeta)
         {
             txJson[jss::metaData] = stMeta->getJson(JsonOptions::none);
+
+            // If applicable, insert delivered amount
             if (txnType == ttPAYMENT || txnType == ttCHECK_CASH)
-            {
-                // Insert delivered amount
-                auto txMeta = std::make_shared<TxMeta>(
-                    txn->getTransactionID(), fill.ledger.seq(), *stMeta);
                 RPC::insertDeliveredAmount(
-                    txJson[jss::metaData], fill.ledger, txn, *txMeta);
-            }
+                    txJson[jss::metaData],
+                    fill.ledger,
+                    txn,
+                    {txn->getTransactionID(), fill.ledger.seq(), *stMeta});
         }
     }
 

src/ripple/app/ledger/impl/SkipListAcquire.cpp

@@ -137,7 +137,7 @@ SkipListAcquire::pmDowncast()
 void
 SkipListAcquire::processData(
     std::uint32_t ledgerSeq,
-    std::shared_ptr<SHAMapItem const> const& item)
+    boost::intrusive_ptr<SHAMapItem const> const& item)
 {
     assert(ledgerSeq != 0 && item);
     ScopedLockType sl(mtx_);

src/ripple/app/ledger/impl/SkipListAcquire.h

@@ -96,7 +96,7 @@ public:
     void
     processData(
         std::uint32_t ledgerSeq,
-        std::shared_ptr<SHAMapItem const> const& item);
+        boost::intrusive_ptr<SHAMapItem const> const& item);
 
     /**
      * Add a callback that will be called when the skipList is ready or failed.

src/ripple/app/ledger/impl/TransactionMaster.cpp

@@ -111,7 +111,7 @@ TransactionMaster::fetch(
 
 std::shared_ptr<STTx const>
 TransactionMaster::fetch(
-    std::shared_ptr<SHAMapItem> const& item,
+    boost::intrusive_ptr<SHAMapItem> const& item,
     SHAMapNodeType type,
     std::uint32_t uCommitLedger)
 {

src/ripple/app/main/Application.cpp

@@ -37,6 +37,7 @@
 #include <ripple/app/main/NodeStoreScheduler.h>
 #include <ripple/app/main/Tuning.h>
 #include <ripple/app/misc/AmendmentTable.h>
+#include <ripple/app/misc/DatagramMonitor.h>
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
 #include <ripple/app/misc/NetworkOPs.h>
@@ -167,6 +168,8 @@ public:
     std::unique_ptr<Logs> logs_;
     std::unique_ptr<TimeKeeper> timeKeeper_;
 
+    std::unique_ptr<DatagramMonitor> datagram_monitor_;
+
     std::uint64_t const instanceCookie_;
 
     beast::Journal m_journal;
@@ -1523,6 +1526,14 @@ ApplicationImp::setup(boost::program_options::variables_map const& cmdline)
     if (reportingETL_)
         reportingETL_->start();
 
+    // Datagram monitor if applicable
+    if (!config_->standalone() && !config_->DATAGRAM_MONITOR.empty())
+    {
+        datagram_monitor_ = std::make_unique<DatagramMonitor>(*this);
+        if (datagram_monitor_)
+            datagram_monitor_->start();
+    }
+
     return true;
 }
 

src/ripple/app/main/Application.h

@@ -19,7 +19,6 @@
 
 #ifndef RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED
 #define RIPPLE_APP_MAIN_APPLICATION_H_INCLUDED
-
 #include <ripple/basics/TaggedCache.h>
 #include <ripple/beast/utility/PropertyStream.h>
 #include <ripple/core/Config.h>

src/ripple/app/main/Main.cpp

@@ -124,14 +124,12 @@ printHelp(const po::options_description& desc)
         << systemName() << "d [options] <command> <params>\n"
         << desc << std::endl
         << "Commands: \n"
-           "     account_currencies <account> [<ledger>] [strict]\n"
-           "     account_info <account>|<seed>|<pass_phrase>|<key> [<ledger>] "
-           "[strict]\n"
+           "     account_currencies <account> [<ledger>]\n"
+           "     account_info <account>|<key> [<ledger>]\n"
            "     account_lines <account> <account>|\"\" [<ledger>]\n"
            "     account_channels <account> <account>|\"\" [<ledger>]\n"
-           "     account_objects <account> [<ledger>] [strict]\n"
-           "     account_offers <account>|<account_public_key> [<ledger>] "
-           "[strict]\n"
+           "     account_objects <account> [<ledger>]\n"
+           "     account_offers <account>|<account_public_key> [<ledger>]\n"
            "     account_tx accountID [ledger_index_min [ledger_index_max "
            "[limit "
            "]]] [binary]\n"

src/ripple/app/misc/AmendmentTable.h

@@ -172,8 +172,7 @@ public:
 
             initialPosition->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(
-                    amendTx.getTransactionID(), s.slice()));
+                make_shamapitem(amendTx.getTransactionID(), s.slice()));
         }
     }
 };

src/ripple/app/misc/FeeVoteImpl.cpp

@@ -326,7 +326,7 @@ FeeVoteImpl::doVoting(
 
         if (!initialPosition->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(journal_.warn()) << "Ledger already had fee change";
         }

src/ripple/app/misc/NegativeUNLVote.cpp

@@ -19,7 +19,10 @@
 
 #include <ripple/app/consensus/RCLValidations.h>
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/main/Application.h>
 #include <ripple/app/misc/NegativeUNLVote.h>
+#include <ripple/json/to_string.h>
+#include <ripple/shamap/SHAMapItem.h>
 
 namespace ripple {
 
@@ -145,7 +148,7 @@ NegativeUNLVote::addReportingTx(
         repUnlTx.add(s);
         if (!initalSet->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(j_.warn()) << "R-UNL: ledger seq=" << seq
                             << ", add ttUNL_REPORT tx failed";
@@ -200,7 +203,7 @@ NegativeUNLVote::addImportVLTx(
         repUnlTx.add(s);
         if (!initalSet->addGiveItem(
                 SHAMapNodeType::tnTRANSACTION_NM,
-                std::make_shared<SHAMapItem>(txID, s.slice())))
+                make_shamapitem(txID, s.slice())))
         {
             JLOG(j_.warn()) << "R-UNL: ledger seq=" << seq
                             << ", add ttUNL_REPORT tx failed (import_vl_key)";
@@ -229,12 +232,11 @@ NegativeUNLVote::addTx(
         obj.setFieldVL(sfUNLModifyValidator, vp.slice());
     });
 
-    uint256 txID = negUnlTx.getTransactionID();
     Serializer s;
     negUnlTx.add(s);
     if (!initialSet->addGiveItem(
             SHAMapNodeType::tnTRANSACTION_NM,
-            std::make_shared<SHAMapItem>(txID, s.slice())))
+            make_shamapitem(negUnlTx.getTransactionID(), s.slice())))
     {
         JLOG(j_.warn()) << "N-UNL: ledger seq=" << seq
                         << ", add ttUNL_MODIFY tx failed";
@@ -242,8 +244,8 @@ NegativeUNLVote::addTx(
     else
     {
         JLOG(j_.debug()) << "N-UNL: ledger seq=" << seq
-                         << ", add a ttUNL_MODIFY Tx with txID: " << txID
-                         << ", the validator to "
+                         << ", add a ttUNL_MODIFY Tx with txID: "
+                         << negUnlTx.getTransactionID() << ", the validator to "
                          << (modify == ToDisable ? "disable: " : "re-enable: ")
                          << vp;
     }

src/ripple/app/misc/NetworkOPs.cpp

@@ -33,6 +33,7 @@
 #include <ripple/app/misc/HashRouter.h>
 #include <ripple/app/misc/LoadFeeTrack.h>
 #include <ripple/app/misc/NetworkOPs.h>
+#include <ripple/app/misc/StateAccounting.h>
 #include <ripple/app/misc/Transaction.h>
 #include <ripple/app/misc/TxQ.h>
 #include <ripple/app/misc/ValidatorKeys.h>
@@ -67,10 +68,12 @@
 #include <ripple/rpc/CTID.h>
 #include <ripple/rpc/DeliveredAmount.h>
 #include <ripple/rpc/impl/RPCHelpers.h>
+#include <ripple/rpc/impl/UDPInfoSub.h>
 #include <boost/asio/ip/host_name.hpp>
 #include <boost/asio/steady_timer.hpp>
-
+#include <exception>
 #include <mutex>
+#include <set>
 #include <string>
 #include <tuple>
 #include <unordered_map>
@@ -114,81 +117,6 @@ class NetworkOPsImp final : public NetworkOPs
         running,
     };
 
-    static std::array<char const*, 5> const states_;
-
-    /**
-     * State accounting records two attributes for each possible server state:
-     * 1) Amount of time spent in each state (in microseconds). This value is
-     *    updated upon each state transition.
-     * 2) Number of transitions to each state.
-     *
-     * This data can be polled through server_info and represented by
-     * monitoring systems similarly to how bandwidth, CPU, and other
-     * counter-based metrics are managed.
-     *
-     * State accounting is more accurate than periodic sampling of server
-     * state. With periodic sampling, it is very likely that state transitions
-     * are missed, and accuracy of time spent in each state is very rough.
-     */
-    class StateAccounting
-    {
-        struct Counters
-        {
-            explicit Counters() = default;
-
-            std::uint64_t transitions = 0;
-            std::chrono::microseconds dur = std::chrono::microseconds(0);
-        };
-
-        OperatingMode mode_ = OperatingMode::DISCONNECTED;
-        std::array<Counters, 5> counters_;
-        mutable std::mutex mutex_;
-        std::chrono::steady_clock::time_point start_ =
-            std::chrono::steady_clock::now();
-        std::chrono::steady_clock::time_point const processStart_ = start_;
-        std::uint64_t initialSyncUs_{0};
-        static std::array<Json::StaticString const, 5> const states_;
-
-    public:
-        explicit StateAccounting()
-        {
-            counters_[static_cast<std::size_t>(OperatingMode::DISCONNECTED)]
-                .transitions = 1;
-        }
-
-        /**
-         * Record state transition. Update duration spent in previous
-         * state.
-         *
-         * @param om New state.
-         */
-        void
-        mode(OperatingMode om);
-
-        /**
-         * Output state counters in JSON format.
-         *
-         * @obj Json object to which to add state accounting data.
-         */
-        void
-        json(Json::Value& obj) const;
-
-        struct CounterData
-        {
-            decltype(counters_) counters;
-            decltype(mode_) mode;
-            decltype(start_) start;
-            decltype(initialSyncUs_) initialSyncUs;
-        };
-
-        CounterData
-        getCounterData() const
-        {
-            std::lock_guard lock(mutex_);
-            return {counters_, mode_, start_, initialSyncUs_};
-        }
-    };
-
     //! Server fees published on `server` subscription
     struct ServerFeeSummary
     {
@@ -270,6 +198,9 @@ public:
     std::string
     strOperatingMode(bool const admin = false) const override;
 
+    StateAccounting::CounterData
+    getStateAccountingData();
+
     //
     // Transaction operations.
     //
@@ -774,7 +705,16 @@ private:
     DispatchState mDispatchState = DispatchState::none;
     std::vector<TransactionStatus> mTransactions;
 
-    StateAccounting accounting_{};
+    StateAccounting accounting_;
+
+    std::set<uint256> pendingValidations_;
+    std::mutex validationsMutex_;
+
+    RCLConsensus&
+    getConsensus();
+
+    LedgerMaster&
+    getLedgerMaster();
 
 private:
     struct Stats
@@ -838,19 +778,6 @@ private:
 
 //------------------------------------------------------------------------------
 
-static std::array<char const*, 5> const stateNames{
-    {"disconnected", "connected", "syncing", "tracking", "full"}};
-
-std::array<char const*, 5> const NetworkOPsImp::states_ = stateNames;
-
-std::array<Json::StaticString const, 5> const
-    NetworkOPsImp::StateAccounting::states_ = {
-        {Json::StaticString(stateNames[0]),
-         Json::StaticString(stateNames[1]),
-         Json::StaticString(stateNames[2]),
-         Json::StaticString(stateNames[3]),
-         Json::StaticString(stateNames[4])}};
-
 static auto const genesisAccountId = calcAccountID(
     generateKeyPair(KeyType::secp256k1, generateSeed("masterpassphrase"))
         .first);
@@ -1125,7 +1052,7 @@ NetworkOPsImp::strOperatingMode(OperatingMode const mode, bool const admin)
         }
     }
 
-    return states_[static_cast<std::size_t>(mode)];
+    return {StateAccounting::states_[static_cast<std::size_t>(mode)].c_str()};
 }
 
 void
@@ -1791,7 +1718,8 @@ NetworkOPsImp::checkLastClosedLedger(
     }
 
     JLOG(m_journal.warn()) << "We are not running on the consensus ledger";
-    JLOG(m_journal.info()) << "Our LCL: " << getJson({*ourClosed, {}});
+    JLOG(m_journal.info()) << "Our LCL: " << ourClosed->info().hash
+                           << getJson({*ourClosed, {}});
     JLOG(m_journal.info()) << "Net LCL " << closedLedger;
 
     if ((mMode == OperatingMode::TRACKING) || (mMode == OperatingMode::FULL))
@@ -2345,7 +2273,37 @@ NetworkOPsImp::recvValidation(
     JLOG(m_journal.trace())
         << "recvValidation " << val->getLedgerHash() << " from " << source;
 
-    handleNewValidation(app_, val, source);
+    // handleNewValidation(app_, val, source);
+    // https://github.com/XRPLF/rippled/commit/fbbea9e6e25795a8a6bd1bf64b780771933a9579
+    std::unique_lock lock(validationsMutex_);
+    BypassAccept bypassAccept = BypassAccept::no;
+    try
+    {
+        if (pendingValidations_.contains(val->getLedgerHash()))
+            bypassAccept = BypassAccept::yes;
+        else
+            pendingValidations_.insert(val->getLedgerHash());
+        lock.unlock();
+        handleNewValidation(app_, val, source, bypassAccept, m_journal);
+    }
+    catch (std::exception const& e)
+    {
+        JLOG(m_journal.warn())
+            << "Exception thrown for handling new validation "
+            << val->getLedgerHash() << ": " << e.what();
+    }
+    catch (...)
+    {
+        JLOG(m_journal.warn())
+            << "Unknown exception thrown for handling new validation "
+            << val->getLedgerHash();
+    }
+    if (bypassAccept == BypassAccept::no)
+    {
+        lock.lock();
+        pendingValidations_.erase(val->getLedgerHash());
+        lock.unlock();
+    }
 
     pubValidation(val);
 
@@ -2360,6 +2318,19 @@ NetworkOPsImp::getConsensusInfo()
     return mConsensus.getJson(true);
 }
 
+// RHTODO: not threadsafe?
+RCLConsensus&
+NetworkOPsImp::getConsensus()
+{
+    return mConsensus;
+}
+
+LedgerMaster&
+NetworkOPsImp::getLedgerMaster()
+{
+    return m_ledgerMaster;
+}
+
 Json::Value
 NetworkOPsImp::getServerInfo(bool human, bool admin, bool counters)
 {
@@ -4157,6 +4128,12 @@ NetworkOPsImp::stateAccounting(Json::Value& obj)
     accounting_.json(obj);
 }
 
+StateAccounting::CounterData
+NetworkOPsImp::getStateAccountingData()
+{
+    return accounting_.getCounterData();
+}
+
 // <-- bool: true=erased, false=was not there
 bool
 NetworkOPsImp::unsubValidations(std::uint64_t uSeq)
@@ -4627,50 +4604,6 @@ NetworkOPsImp::collect_metrics()
         counters[static_cast<std::size_t>(OperatingMode::FULL)].transitions);
 }
 
-void
-NetworkOPsImp::StateAccounting::mode(OperatingMode om)
-{
-    auto now = std::chrono::steady_clock::now();
-
-    std::lock_guard lock(mutex_);
-    ++counters_[static_cast<std::size_t>(om)].transitions;
-    if (om == OperatingMode::FULL &&
-        counters_[static_cast<std::size_t>(om)].transitions == 1)
-    {
-        initialSyncUs_ = std::chrono::duration_cast<std::chrono::microseconds>(
-                             now - processStart_)
-                             .count();
-    }
-    counters_[static_cast<std::size_t>(mode_)].dur +=
-        std::chrono::duration_cast<std::chrono::microseconds>(now - start_);
-
-    mode_ = om;
-    start_ = now;
-}
-
-void
-NetworkOPsImp::StateAccounting::json(Json::Value& obj) const
-{
-    auto [counters, mode, start, initialSync] = getCounterData();
-    auto const current = std::chrono::duration_cast<std::chrono::microseconds>(
-        std::chrono::steady_clock::now() - start);
-    counters[static_cast<std::size_t>(mode)].dur += current;
-
-    obj[jss::state_accounting] = Json::objectValue;
-    for (std::size_t i = static_cast<std::size_t>(OperatingMode::DISCONNECTED);
-         i <= static_cast<std::size_t>(OperatingMode::FULL);
-         ++i)
-    {
-        obj[jss::state_accounting][states_[i]] = Json::objectValue;
-        auto& state = obj[jss::state_accounting][states_[i]];
-        state[jss::transitions] = std::to_string(counters[i].transitions);
-        state[jss::duration_us] = std::to_string(counters[i].dur.count());
-    }
-    obj[jss::server_state_duration_us] = std::to_string(current.count());
-    if (initialSync)
-        obj[jss::initial_sync_duration_us] = std::to_string(initialSync);
-}
-
 //------------------------------------------------------------------------------
 
 std::unique_ptr<NetworkOPs>

src/ripple/app/misc/NetworkOPs.h

@@ -20,8 +20,10 @@
 #ifndef RIPPLE_APP_MISC_NETWORKOPS_H_INCLUDED
 #define RIPPLE_APP_MISC_NETWORKOPS_H_INCLUDED
 
+#include <ripple/app/consensus/RCLConsensus.h>
 #include <ripple/app/consensus/RCLCxPeerPos.h>
 #include <ripple/app/ledger/Ledger.h>
+#include <ripple/app/misc/StateAccounting.h>
 #include <ripple/core/JobQueue.h>
 #include <ripple/ledger/ReadView.h>
 #include <ripple/net/InfoSub.h>
@@ -42,35 +44,6 @@ class LedgerMaster;
 class Transaction;
 class ValidatorKeys;
 
-// This is the primary interface into the "client" portion of the program.
-// Code that wants to do normal operations on the network such as
-// creating and monitoring accounts, creating transactions, and so on
-// should use this interface. The RPC code will primarily be a light wrapper
-// over this code.
-//
-// Eventually, it will check the node's operating mode (synched, unsynched,
-// etectera) and defer to the correct means of processing. The current
-// code assumes this node is synched (and will continue to do so until
-// there's a functional network.
-//
-
-/** Specifies the mode under which the server believes it's operating.
-
-    This has implications about how the server processes transactions and
-    how it responds to requests (e.g. account balance request).
-
-    @note Other code relies on the numerical values of these constants; do
-          not change them without verifying each use and ensuring that it is
-          not a breaking change.
-*/
-enum class OperatingMode {
-    DISCONNECTED = 0,  //!< not ready to process requests
-    CONNECTED = 1,     //!< convinced we are talking to the network
-    SYNCING = 2,       //!< fallen slightly behind
-    TRACKING = 3,      //!< convinced we agree with the network
-    FULL = 4           //!< we have the ledger and can even validate
-};
-
 /** Provides server functionality for clients.
 
     Clients include backend applications, local commands, and connected
@@ -221,6 +194,13 @@ public:
 
     virtual Json::Value
     getConsensusInfo() = 0;
+
+    virtual RCLConsensus&
+    getConsensus() = 0;
+
+    virtual LedgerMaster&
+    getLedgerMaster() = 0;
+
     virtual Json::Value
     getServerInfo(bool human, bool admin, bool counters) = 0;
     virtual void
@@ -228,6 +208,9 @@ public:
     virtual Json::Value
     getLedgerFetchInfo() = 0;
 
+    virtual StateAccounting::CounterData
+    getStateAccountingData() = 0;
+
     /** Accepts the current transaction tree, return the new ledger's sequence
 
         This API is only used via RPC with the server in STANDALONE mode and

src/ripple/app/misc/SHAMapStoreImp.cpp

@@ -162,7 +162,8 @@ SHAMapStoreImp::SHAMapStoreImp(
         }
 
         state_db_.init(config, dbName_);
-        dbPaths();
+        if (!config.mem_backend())
+            dbPaths();
     }
 }
 
@@ -195,6 +196,7 @@ SHAMapStoreImp::makeNodeStore(int readThreads)
                 "online_delete info from config");
         }
         SavedState state = state_db_.getState();
+
         auto writableBackend = makeBackendRotating(state.writableDb);
         auto archiveBackend = makeBackendRotating(state.archiveDb);
         if (!state.writableDb.size())
@@ -293,6 +295,8 @@ SHAMapStoreImp::run()
     fullBelowCache_ = &(*app_.getNodeFamily().getFullBelowCache(0));
     treeNodeCache_ = &(*app_.getNodeFamily().getTreeNodeCache(0));
 
+    bool const isMem = app_.config().mem_backend();
+
     if (advisoryDelete_)
         canDelete_ = state_db_.getCanDelete();
 
@@ -351,7 +355,7 @@ SHAMapStoreImp::run()
         // will delete up to (not including) lastRotated
         if (readyToRotate && !waitForImport)
         {
-            JLOG(journal_.warn())
+            JLOG(journal_.debug())
                 << "rotating  validatedSeq " << validatedSeq << " lastRotated "
                 << lastRotated << " deleteInterval " << deleteInterval_
                 << " canDelete_ " << canDelete_ << " state "
@@ -395,7 +399,7 @@ SHAMapStoreImp::run()
             // Only log if we completed without a "health" abort
             JLOG(journal_.debug()) << validatedSeq << " freshened caches";
 
-            JLOG(journal_.trace()) << "Making a new backend";
+            JLOG(journal_.debug()) << "Making a new backend";
             auto newBackend = makeBackendRotating();
             JLOG(journal_.debug())
                 << validatedSeq << " new backend " << newBackend->getName();
@@ -640,6 +644,33 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
     if (!db)
         Throw<std::runtime_error>("Failed to get relational database");
 
+    if (app_.config().useTxTables())
+    {
+        clearSql(
+            lastRotated,
+            "Transactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+
+        clearSql(
+            lastRotated,
+            "AccountTransactions",
+            [&db]() -> std::optional<LedgerIndex> {
+                return db->getAccountTransactionsMinLedgerSeq();
+            },
+            [&db](LedgerIndex min) -> void {
+                db->deleteAccountTransactionsBeforeLedgerSeq(min);
+            });
+        if (healthWait() == stopping)
+            return;
+    }
+
     clearSql(
         lastRotated,
         "Ledgers",
@@ -647,33 +678,6 @@ SHAMapStoreImp::clearPrior(LedgerIndex lastRotated)
         [db](LedgerIndex min) -> void { db->deleteBeforeLedgerSeq(min); });
     if (healthWait() == stopping)
         return;
-
-    if (!app_.config().useTxTables())
-        return;
-
-    clearSql(
-        lastRotated,
-        "Transactions",
-        [&db]() -> std::optional<LedgerIndex> {
-            return db->getTransactionsMinLedgerSeq();
-        },
-        [&db](LedgerIndex min) -> void {
-            db->deleteTransactionsBeforeLedgerSeq(min);
-        });
-    if (healthWait() == stopping)
-        return;
-
-    clearSql(
-        lastRotated,
-        "AccountTransactions",
-        [&db]() -> std::optional<LedgerIndex> {
-            return db->getAccountTransactionsMinLedgerSeq();
-        },
-        [&db](LedgerIndex min) -> void {
-            db->deleteAccountTransactionsBeforeLedgerSeq(min);
-        });
-    if (healthWait() == stopping)
-        return;
 }
 
 SHAMapStoreImp::HealthResult

src/ripple/app/misc/StateAccounting.cpp

@@ -0,0 +1,49 @@
+#include <ripple/app/misc/StateAccounting.h>
+
+namespace ripple {
+
+void
+StateAccounting::mode(OperatingMode om)
+{
+    std::lock_guard lock(mutex_);
+    auto now = std::chrono::steady_clock::now();
+
+    ++counters_[static_cast<std::size_t>(om)].transitions;
+    if (om == OperatingMode::FULL &&
+        counters_[static_cast<std::size_t>(om)].transitions == 1)
+    {
+        initialSyncUs_ = std::chrono::duration_cast<std::chrono::microseconds>(
+                             now - processStart_)
+                             .count();
+    }
+    counters_[static_cast<std::size_t>(mode_)].dur +=
+        std::chrono::duration_cast<std::chrono::microseconds>(now - start_);
+
+    mode_ = om;
+    start_ = now;
+}
+
+void
+StateAccounting::json(Json::Value& obj)
+{
+    auto [counters, mode, start, initialSync] = getCounterData();
+    auto const current = std::chrono::duration_cast<std::chrono::microseconds>(
+        std::chrono::steady_clock::now() - start);
+    counters[static_cast<std::size_t>(mode)].dur += current;
+
+    obj[jss::state_accounting] = Json::objectValue;
+    for (std::size_t i = static_cast<std::size_t>(OperatingMode::DISCONNECTED);
+         i <= static_cast<std::size_t>(OperatingMode::FULL);
+         ++i)
+    {
+        obj[jss::state_accounting][states_[i]] = Json::objectValue;
+        auto& state = obj[jss::state_accounting][states_[i]];
+        state[jss::transitions] = std::to_string(counters[i].transitions);
+        state[jss::duration_us] = std::to_string(counters[i].dur.count());
+    }
+    obj[jss::server_state_duration_us] = std::to_string(current.count());
+    if (initialSync)
+        obj[jss::initial_sync_duration_us] = std::to_string(initialSync);
+}
+
+}  // namespace ripple

src/ripple/app/misc/StateAccounting.h

@@ -0,0 +1,99 @@
+#ifndef RIPPLE_APP_MAIN_STATEACCOUNTING_H_INCLUDED
+#define RIPPLE_APP_MAIN_STATEACCOUNTING_H_INCLUDED
+
+#include <ripple/basics/chrono.h>
+#include <ripple/beast/utility/Journal.h>
+#include <ripple/json/json_value.h>
+#include <ripple/protocol/jss.h>
+#include <array>
+#include <mutex>
+
+namespace ripple {
+
+// This is the primary interface into the "client" portion of the program.
+// Code that wants to do normal operations on the network such as
+// creating and monitoring accounts, creating transactions, and so on
+// should use this interface. The RPC code will primarily be a light wrapper
+// over this code.
+//
+// Eventually, it will check the node's operating mode (synched, unsynched,
+// etectera) and defer to the correct means of processing. The current
+// code assumes this node is synched (and will continue to do so until
+// there's a functional network.
+//
+
+/** Specifies the mode under which the server believes it's operating.
+
+    This has implications about how the server processes transactions and
+    how it responds to requests (e.g. account balance request).
+
+    @note Other code relies on the numerical values of these constants; do
+          not change them without verifying each use and ensuring that it is
+          not a breaking change.
+*/
+enum class OperatingMode {
+    DISCONNECTED = 0,  //!< not ready to process requests
+    CONNECTED = 1,     //!< convinced we are talking to the network
+    SYNCING = 2,       //!< fallen slightly behind
+    TRACKING = 3,      //!< convinced we agree with the network
+    FULL = 4           //!< we have the ledger and can even validate
+};
+
+class StateAccounting
+{
+public:
+    constexpr static std::array<Json::StaticString const, 5> const states_ = {
+        {Json::StaticString("disconnected"),
+         Json::StaticString("connected"),
+         Json::StaticString("syncing"),
+         Json::StaticString("tracking"),
+         Json::StaticString("full")}};
+
+    struct Counters
+    {
+        explicit Counters() = default;
+
+        std::uint64_t transitions = 0;
+        std::chrono::microseconds dur = std::chrono::microseconds(0);
+    };
+
+private:
+    OperatingMode mode_ = OperatingMode::DISCONNECTED;
+    std::array<Counters, 5> counters_;
+    mutable std::mutex mutex_;
+    std::chrono::steady_clock::time_point start_ =
+        std::chrono::steady_clock::now();
+    std::chrono::steady_clock::time_point const processStart_ = start_;
+    std::uint64_t initialSyncUs_{0};
+
+public:
+    explicit StateAccounting()
+    {
+        counters_[static_cast<std::size_t>(OperatingMode::DISCONNECTED)]
+            .transitions = 1;
+    }
+
+    //! Record state transition. Update duration spent in previous state.
+    void
+    mode(OperatingMode om);
+
+    //! Output state counters in JSON format.
+    void
+    json(Json::Value& obj);
+
+    using CounterData = std::tuple<
+        decltype(counters_),
+        decltype(mode_),
+        decltype(start_),
+        decltype(initialSyncUs_)>;
+
+    CounterData
+    getCounterData()
+    {
+        return {counters_, mode_, start_, initialSyncUs_};
+    }
+};
+
+}  // namespace ripple
+
+#endif

src/ripple/app/misc/TxQ.h

@@ -111,7 +111,7 @@ public:
         std::uint32_t minimumTxnInLedgerSA = 1000;
         /// Number of transactions per ledger that fee escalation "works
         /// towards".
-        std::uint32_t targetTxnInLedger = 256;
+        std::uint32_t targetTxnInLedger = 1000;
         /** Optional maximum allowed value of transactions per ledger before
             fee escalation kicks in. By default, the maximum is an emergent
             property of network, validator, and consensus performance. This

src/ripple/app/misc/impl/ValidatorList.cpp

@@ -1710,6 +1710,15 @@ ValidatorList::calculateQuorum(
     std::size_t effectiveUnlSize,
     std::size_t seenSize)
 {
+    // Use quorum if specified via command line.
+    if (minimumQuorum_ > 0)
+    {
+        JLOG(j_.warn()) << "Using potentially unsafe quorum of "
+                        << *minimumQuorum_
+                        << " as specified on the command line";
+        return *minimumQuorum_;
+    }
+
     // Do not use achievable quorum until lists from all configured
     // publishers are available
     for (auto const& list : publisherLists_)
@@ -1752,21 +1761,8 @@ ValidatorList::calculateQuorum(
     // Note that the negative UNL protocol introduced the
     // AbsoluteMinimumQuorum which is 60% of the original UNL size. The
     // effective quorum should not be lower than it.
-    auto quorum = static_cast<std::size_t>(std::max(
+    return static_cast<std::size_t>(std::max(
         std::ceil(effectiveUnlSize * 0.8f), std::ceil(unlSize * 0.6f)));
-
-    // Use lower quorum specified via command line if the normal quorum
-    // appears unreachable based on the number of recently received
-    // validations.
-    if (minimumQuorum_)  // < quorum && seenSize < quorum)
-    {
-        quorum = *minimumQuorum_;
-
-        JLOG(j_.warn()) << "Using unsafe quorum of " << quorum
-                        << " as specified in the command line";
-    }
-
-    return quorum;
 }
 
 TrustChanges

src/ripple/app/rdb/backend/FlatmapDatabase.h

@@ -0,0 +1,853 @@
+#ifndef RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED
+#define RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED
+
+#include <ripple/app/ledger/AcceptedLedger.h>
+#include <ripple/app/ledger/LedgerMaster.h>
+#include <ripple/app/ledger/TransactionMaster.h>
+#include <ripple/app/rdb/backend/SQLiteDatabase.h>
+#include <algorithm>
+#include <map>
+#include <mutex>
+#include <optional>
+#include <shared_mutex>
+#include <vector>
+
+#include <boost/unordered/concurrent_flat_map.hpp>
+
+namespace ripple {
+
+struct base_uint_hasher
+{
+    using result_type = std::size_t;
+
+    result_type
+    operator()(base_uint<256> const& value) const
+    {
+        return hardened_hash<>{}(value);
+    }
+
+    result_type
+    operator()(AccountID const& value) const
+    {
+        return hardened_hash<>{}(value);
+    }
+};
+
+class FlatmapDatabase : public SQLiteDatabase
+{
+private:
+    struct LedgerData
+    {
+        LedgerInfo info;
+        boost::unordered::
+            concurrent_flat_map<uint256, AccountTx, base_uint_hasher>
+                transactions;
+    };
+
+    struct AccountTxData
+    {
+        boost::unordered::
+            concurrent_flat_map<std::pair<uint32_t, uint32_t>, AccountTx>
+                transactions;
+    };
+
+    Application& app_;
+    Config const& config_;
+    JobQueue& jobQueue_;
+
+    boost::unordered::concurrent_flat_map<LedgerIndex, LedgerData> ledgers_;
+    boost::unordered::
+        concurrent_flat_map<uint256, LedgerIndex, base_uint_hasher>
+            ledgerHashToSeq_;
+    boost::unordered::concurrent_flat_map<uint256, AccountTx, base_uint_hasher>
+        transactionMap_;
+    boost::unordered::
+        concurrent_flat_map<AccountID, AccountTxData, base_uint_hasher>
+            accountTxMap_;
+
+public:
+    FlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue)
+        : app_(app), config_(config), jobQueue_(jobQueue)
+    {
+    }
+
+    std::optional<LedgerIndex>
+    getMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        ledgers_.visit_all([&minSeq](auto const& pair) {
+            if (!minSeq || pair.first < *minSeq)
+            {
+                minSeq = pair.first;
+            }
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getTransactionsMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        transactionMap_.visit_all([&minSeq](auto const& pair) {
+            LedgerIndex seq = pair.second.second->getLgrSeq();
+            if (!minSeq || seq < *minSeq)
+            {
+                minSeq = seq;
+            }
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getAccountTransactionsMinLedgerSeq() override
+    {
+        std::optional<LedgerIndex> minSeq;
+        accountTxMap_.visit_all([&minSeq](auto const& pair) {
+            pair.second.transactions.visit_all([&minSeq](auto const& tx) {
+                if (!minSeq || tx.first.first < *minSeq)
+                {
+                    minSeq = tx.first.first;
+                }
+            });
+        });
+        return minSeq;
+    }
+
+    std::optional<LedgerIndex>
+    getMaxLedgerSeq() override
+    {
+        std::optional<LedgerIndex> maxSeq;
+        ledgers_.visit_all([&maxSeq](auto const& pair) {
+            if (!maxSeq || pair.first > *maxSeq)
+            {
+                maxSeq = pair.first;
+            }
+        });
+        return maxSeq;
+    }
+    void
+    deleteTransactionByLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.visit(ledgerSeq, [this](auto& item) {
+            item.second.transactions.visit_all([this](auto const& txPair) {
+                transactionMap_.erase(txPair.first);
+            });
+            item.second.transactions.clear();
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first == ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.erase_if([this, ledgerSeq](auto const& item) {
+            if (item.first < ledgerSeq)
+            {
+                item.second.transactions.visit_all([this](auto const& txPair) {
+                    transactionMap_.erase(txPair.first);
+                });
+                ledgerHashToSeq_.erase(item.second.info.hash);
+                return true;
+            }
+            return false;
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        ledgers_.visit_all([this, ledgerSeq](auto& item) {
+            if (item.first < ledgerSeq)
+            {
+                item.second.transactions.visit_all([this](auto const& txPair) {
+                    transactionMap_.erase(txPair.first);
+                });
+                item.second.transactions.clear();
+            }
+        });
+
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+
+    void
+    deleteAccountTransactionsBeforeLedgerSeq(LedgerIndex ledgerSeq) override
+    {
+        accountTxMap_.visit_all([ledgerSeq](auto& item) {
+            item.second.transactions.erase_if([ledgerSeq](auto const& tx) {
+                return tx.first.first < ledgerSeq;
+            });
+        });
+    }
+    std::size_t
+    getTransactionCount() override
+    {
+        return transactionMap_.size();
+    }
+
+    std::size_t
+    getAccountTransactionCount() override
+    {
+        std::size_t count = 0;
+        accountTxMap_.visit_all([&count](auto const& item) {
+            count += item.second.transactions.size();
+        });
+        return count;
+    }
+
+    CountMinMax
+    getLedgerCountMinMax() override
+    {
+        CountMinMax result{0, 0, 0};
+        ledgers_.visit_all([&result](auto const& item) {
+            result.numberOfRows++;
+            if (result.minLedgerSequence == 0 ||
+                item.first < result.minLedgerSequence)
+            {
+                result.minLedgerSequence = item.first;
+            }
+            if (item.first > result.maxLedgerSequence)
+            {
+                result.maxLedgerSequence = item.first;
+            }
+        });
+        return result;
+    }
+
+    bool
+    saveValidatedLedger(
+        std::shared_ptr<Ledger const> const& ledger,
+        bool current) override
+    {
+        try
+        {
+            LedgerData ledgerData;
+            ledgerData.info = ledger->info();
+
+            auto aLedger = std::make_shared<AcceptedLedger>(ledger, app_);
+            for (auto const& acceptedLedgerTx : *aLedger)
+            {
+                auto const& txn = acceptedLedgerTx->getTxn();
+                auto const& meta = acceptedLedgerTx->getMeta();
+                auto const& id = txn->getTransactionID();
+
+                std::string reason;
+                auto accTx = std::make_pair(
+                    std::make_shared<ripple::Transaction>(txn, reason, app_),
+                    std::make_shared<ripple::TxMeta>(meta));
+
+                ledgerData.transactions.emplace(id, accTx);
+                transactionMap_.emplace(id, accTx);
+
+                for (auto const& account : meta.getAffectedAccounts())
+                {
+                    accountTxMap_.visit(account, [&](auto& data) {
+                        data.second.transactions.emplace(
+                            std::make_pair(
+                                ledger->info().seq,
+                                acceptedLedgerTx->getTxnSeq()),
+                            accTx);
+                    });
+                }
+            }
+
+            ledgers_.emplace(ledger->info().seq, std::move(ledgerData));
+            ledgerHashToSeq_.emplace(ledger->info().hash, ledger->info().seq);
+
+            if (current)
+            {
+                auto const cutoffSeq =
+                    ledger->info().seq > app_.config().LEDGER_HISTORY
+                    ? ledger->info().seq - app_.config().LEDGER_HISTORY
+                    : 0;
+
+                if (cutoffSeq > 0)
+                {
+                    const std::size_t BATCH_SIZE = 128;
+                    std::size_t deleted = 0;
+
+                    ledgers_.erase_if([&](auto const& item) {
+                        if (deleted >= BATCH_SIZE)
+                            return false;
+
+                        if (item.first < cutoffSeq)
+                        {
+                            item.second.transactions.visit_all(
+                                [this](auto const& txPair) {
+                                    transactionMap_.erase(txPair.first);
+                                });
+                            ledgerHashToSeq_.erase(item.second.info.hash);
+                            deleted++;
+                            return true;
+                        }
+                        return false;
+                    });
+
+                    if (deleted > 0)
+                    {
+                        accountTxMap_.visit_all([cutoffSeq](auto& item) {
+                            item.second.transactions.erase_if(
+                                [cutoffSeq](auto const& tx) {
+                                    return tx.first.first < cutoffSeq;
+                                });
+                        });
+                    }
+
+                    app_.getLedgerMaster().clearPriorLedgers(cutoffSeq);
+                }
+            }
+
+            return true;
+        }
+        catch (std::exception const&)
+        {
+            deleteTransactionByLedgerSeq(ledger->info().seq);
+            return false;
+        }
+    }
+
+    std::optional<LedgerInfo>
+    getLedgerInfoByIndex(LedgerIndex ledgerSeq) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit(ledgerSeq, [&result](auto const& item) {
+            result = item.second.info;
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getNewestLedgerInfo() override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&result](auto const& item) {
+            if (!result || item.second.info.seq > result->seq)
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLimitedOldestLedgerInfo(LedgerIndex ledgerFirstIndex) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= ledgerFirstIndex &&
+                (!result || item.first < result->seq))
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLimitedNewestLedgerInfo(LedgerIndex ledgerFirstIndex) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= ledgerFirstIndex &&
+                (!result || item.first > result->seq))
+            {
+                result = item.second.info;
+            }
+        });
+        return result;
+    }
+
+    std::optional<LedgerInfo>
+    getLedgerInfoByHash(uint256 const& ledgerHash) override
+    {
+        std::optional<LedgerInfo> result;
+        ledgerHashToSeq_.visit(ledgerHash, [this, &result](auto const& item) {
+            ledgers_.visit(item.second, [&result](auto const& item) {
+                result = item.second.info;
+            });
+        });
+        return result;
+    }
+    uint256
+    getHashByIndex(LedgerIndex ledgerIndex) override
+    {
+        uint256 result;
+        ledgers_.visit(ledgerIndex, [&result](auto const& item) {
+            result = item.second.info.hash;
+        });
+        return result;
+    }
+
+    std::optional<LedgerHashPair>
+    getHashesByIndex(LedgerIndex ledgerIndex) override
+    {
+        std::optional<LedgerHashPair> result;
+        ledgers_.visit(ledgerIndex, [&result](auto const& item) {
+            result = LedgerHashPair{
+                item.second.info.hash, item.second.info.parentHash};
+        });
+        return result;
+    }
+
+    std::map<LedgerIndex, LedgerHashPair>
+    getHashesByIndex(LedgerIndex minSeq, LedgerIndex maxSeq) override
+    {
+        std::map<LedgerIndex, LedgerHashPair> result;
+        ledgers_.visit_all([&](auto const& item) {
+            if (item.first >= minSeq && item.first <= maxSeq)
+            {
+                result[item.first] = LedgerHashPair{
+                    item.second.info.hash, item.second.info.parentHash};
+            }
+        });
+        return result;
+    }
+
+    std::variant<AccountTx, TxSearched>
+    getTransaction(
+        uint256 const& id,
+        std::optional<ClosedInterval<std::uint32_t>> const& range,
+        error_code_i& ec) override
+    {
+        std::variant<AccountTx, TxSearched> result = TxSearched::unknown;
+        transactionMap_.visit(id, [&](auto const& item) {
+            auto const& tx = item.second;
+            if (!range ||
+                (range->lower() <= tx.second->getLgrSeq() &&
+                 tx.second->getLgrSeq() <= range->upper()))
+            {
+                result = tx;
+            }
+            else
+            {
+                result = TxSearched::all;
+            }
+        });
+        return result;
+    }
+
+    bool
+    ledgerDbHasSpace(Config const& config) override
+    {
+        return true;  // In-memory database always has space
+    }
+
+    bool
+    transactionDbHasSpace(Config const& config) override
+    {
+        return true;  // In-memory database always has space
+    }
+
+    std::uint32_t
+    getKBUsedAll() override
+    {
+        std::uint32_t size = sizeof(*this);
+        size += ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData));
+        size +=
+            ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex));
+        size += transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx));
+        accountTxMap_.visit_all([&size](auto const& item) {
+            size += sizeof(AccountID) + sizeof(AccountTxData);
+            size += item.second.transactions.size() * sizeof(AccountTx);
+        });
+        return size / 1024;  // Convert to KB
+    }
+
+    std::uint32_t
+    getKBUsedLedger() override
+    {
+        std::uint32_t size =
+            ledgers_.size() * (sizeof(LedgerIndex) + sizeof(LedgerData));
+        size +=
+            ledgerHashToSeq_.size() * (sizeof(uint256) + sizeof(LedgerIndex));
+        return size / 1024;
+    }
+
+    std::uint32_t
+    getKBUsedTransaction() override
+    {
+        std::uint32_t size =
+            transactionMap_.size() * (sizeof(uint256) + sizeof(AccountTx));
+        accountTxMap_.visit_all([&size](auto const& item) {
+            size += sizeof(AccountID) + sizeof(AccountTxData);
+            size += item.second.transactions.size() * sizeof(AccountTx);
+        });
+        return size / 1024;
+    }
+
+    void
+    closeLedgerDB() override
+    {
+        // No-op for in-memory database
+    }
+
+    void
+    closeTransactionDB() override
+    {
+        // No-op for in-memory database
+    }
+
+    ~FlatmapDatabase()
+    {
+        // Concurrent maps need visit_all
+        accountTxMap_.visit_all(
+            [](auto& pair) { pair.second.transactions.clear(); });
+        accountTxMap_.clear();
+
+        transactionMap_.clear();
+
+        ledgers_.visit_all(
+            [](auto& pair) { pair.second.transactions.clear(); });
+        ledgers_.clear();
+
+        ledgerHashToSeq_.clear();
+    }
+
+    std::vector<std::shared_ptr<Transaction>>
+    getTxHistory(LedgerIndex startIndex) override
+    {
+        std::vector<std::shared_ptr<Transaction>> result;
+        transactionMap_.visit_all([&](auto const& item) {
+            if (item.second.second->getLgrSeq() >= startIndex)
+            {
+                result.push_back(item.second.first);
+            }
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a->getLedger() > b->getLedger();
+            });
+        if (result.size() > 20)
+        {
+            result.resize(20);
+        }
+        return result;
+    }
+    // Helper function to handle limits
+    template <typename Container>
+    void
+    applyLimit(Container& container, std::size_t limit, bool bUnlimited)
+    {
+        if (!bUnlimited && limit > 0 && container.size() > limit)
+        {
+            container.resize(limit);
+        }
+    }
+
+    AccountTxs
+    getOldestAccountTxs(AccountTxOptions const& options) override
+    {
+        AccountTxs result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.push_back(tx.second);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a.second->getLgrSeq() < b.second->getLgrSeq();
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    AccountTxs
+    getNewestAccountTxs(AccountTxOptions const& options) override
+    {
+        AccountTxs result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.push_back(tx.second);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return a.second->getLgrSeq() > b.second->getLgrSeq();
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    MetaTxsList
+    getOldestAccountTxsB(AccountTxOptions const& options) override
+    {
+        MetaTxsList result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.emplace_back(
+                        tx.second.first->getSTransaction()
+                            ->getSerializer()
+                            .peekData(),
+                        tx.second.second->getAsObject()
+                            .getSerializer()
+                            .peekData(),
+                        tx.first.first);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return std::get<2>(a) < std::get<2>(b);
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+
+    MetaTxsList
+    getNewestAccountTxsB(AccountTxOptions const& options) override
+    {
+        MetaTxsList result;
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    result.emplace_back(
+                        tx.second.first->getSTransaction()
+                            ->getSerializer()
+                            .peekData(),
+                        tx.second.second->getAsObject()
+                            .getSerializer()
+                            .peekData(),
+                        tx.first.first);
+                }
+            });
+        });
+        std::sort(
+            result.begin(), result.end(), [](auto const& a, auto const& b) {
+                return std::get<2>(a) > std::get<2>(b);
+            });
+        applyLimit(result, options.limit, options.bUnlimited);
+        return result;
+    }
+    std::pair<AccountTxs, std::optional<AccountTxMarker>>
+    oldestAccountTxPage(AccountTxPageOptions const& options) override
+    {
+        AccountTxs result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::pair<std::pair<uint32_t, uint32_t>, AccountTx>>
+                txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(tx);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) {
+                return a.first < b.first;
+            });
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return tx.first.first == options.marker->ledgerSeq &&
+                        tx.first.second == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                result.push_back(it->second);
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{it->first.first, it->first.second};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<AccountTxs, std::optional<AccountTxMarker>>
+    newestAccountTxPage(AccountTxPageOptions const& options) override
+    {
+        AccountTxs result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::pair<std::pair<uint32_t, uint32_t>, AccountTx>>
+                txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(tx);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), [](auto const& a, auto const& b) {
+                return a.first > b.first;
+            });
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return tx.first.first == options.marker->ledgerSeq &&
+                        tx.first.second == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                result.push_back(it->second);
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{it->first.first, it->first.second};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<MetaTxsList, std::optional<AccountTxMarker>>
+    oldestAccountTxPageB(AccountTxPageOptions const& options) override
+    {
+        MetaTxsList result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::tuple<uint32_t, uint32_t, AccountTx>> txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(
+                        tx.first.first, tx.first.second, tx.second);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end());
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return std::get<0>(tx) == options.marker->ledgerSeq &&
+                        std::get<1>(tx) == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                const auto& [_, __, tx] = *it;
+                result.emplace_back(
+                    tx.first->getSTransaction()->getSerializer().peekData(),
+                    tx.second->getAsObject().getSerializer().peekData(),
+                    std::get<0>(*it));
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)};
+            }
+        });
+
+        return {result, marker};
+    }
+
+    std::pair<MetaTxsList, std::optional<AccountTxMarker>>
+    newestAccountTxPageB(AccountTxPageOptions const& options) override
+    {
+        MetaTxsList result;
+        std::optional<AccountTxMarker> marker;
+
+        accountTxMap_.visit(options.account, [&](auto const& item) {
+            std::vector<std::tuple<uint32_t, uint32_t, AccountTx>> txs;
+            item.second.transactions.visit_all([&](auto const& tx) {
+                if (tx.first.first >= options.minLedger &&
+                    tx.first.first <= options.maxLedger)
+                {
+                    txs.emplace_back(
+                        tx.first.first, tx.first.second, tx.second);
+                }
+            });
+
+            std::sort(txs.begin(), txs.end(), std::greater<>());
+
+            auto it = txs.begin();
+            if (options.marker)
+            {
+                it = std::find_if(txs.begin(), txs.end(), [&](auto const& tx) {
+                    return std::get<0>(tx) == options.marker->ledgerSeq &&
+                        std::get<1>(tx) == options.marker->txnSeq;
+                });
+                if (it != txs.end())
+                    ++it;
+            }
+
+            for (; it != txs.end() &&
+                 (options.limit == 0 || result.size() < options.limit);
+                 ++it)
+            {
+                const auto& [_, __, tx] = *it;
+                result.emplace_back(
+                    tx.first->getSTransaction()->getSerializer().peekData(),
+                    tx.second->getAsObject().getSerializer().peekData(),
+                    std::get<0>(*it));
+            }
+
+            if (it != txs.end())
+            {
+                marker = AccountTxMarker{std::get<0>(*it), std::get<1>(*it)};
+            }
+        });
+
+        return {result, marker};
+    }
+};
+
+// Factory function
+std::unique_ptr<SQLiteDatabase>
+getFlatmapDatabase(Application& app, Config const& config, JobQueue& jobQueue)
+{
+    return std::make_unique<FlatmapDatabase>(app, config, jobQueue);
+}
+
+}  // namespace ripple
+#endif  // RIPPLE_APP_RDB_BACKEND_FLATMAPDATABASE_H_INCLUDED

src/ripple/app/rdb/backend/detail/impl/Node.cpp

@@ -1132,7 +1132,7 @@ accountTxPage(
     {
         sql = boost::str(
             boost::format(
-                prefix + (R"(AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u'
+                prefix + (R"(AccountTransactions.LedgerSeq BETWEEN %u AND %u
              ORDER BY AccountTransactions.LedgerSeq %s,
              AccountTransactions.TxnSeq %s
              LIMIT %u;)")) %
@@ -1155,12 +1155,14 @@ accountTxPage(
             FROM AccountTransactions, Transactions WHERE
             (AccountTransactions.TransID = Transactions.TransID AND
             AccountTransactions.Account = '%s' AND
-            AccountTransactions.LedgerSeq BETWEEN '%u' AND '%u')
-            OR
+            AccountTransactions.LedgerSeq BETWEEN %u AND %u)
+            UNION
+            SELECT AccountTransactions.LedgerSeq,AccountTransactions.TxnSeq,Status,RawTxn,TxnMeta
+            FROM AccountTransactions, Transactions WHERE
             (AccountTransactions.TransID = Transactions.TransID AND
             AccountTransactions.Account = '%s' AND
-            AccountTransactions.LedgerSeq = '%u' AND
-            AccountTransactions.TxnSeq %s '%u')
+            AccountTransactions.LedgerSeq = %u AND
+            AccountTransactions.TxnSeq %s %u)
             ORDER BY AccountTransactions.LedgerSeq %s,
             AccountTransactions.TxnSeq %s
             LIMIT %u;

src/ripple/app/rdb/impl/RelationalDatabase.cpp

@@ -19,6 +19,8 @@
 
 #include <ripple/app/main/Application.h>
 #include <ripple/app/rdb/RelationalDatabase.h>
+#include <ripple/app/rdb/backend/FlatmapDatabase.h>
+#include <ripple/app/rdb/backend/RWDBDatabase.h>
 #include <ripple/core/ConfigSections.h>
 #include <ripple/nodestore/DatabaseShard.h>
 
@@ -38,6 +40,8 @@ RelationalDatabase::init(
 {
     bool use_sqlite = false;
     bool use_postgres = false;
+    bool use_rwdb = false;
+    bool use_flatmap = false;
 
     if (config.reporting())
     {
@@ -52,6 +56,14 @@ RelationalDatabase::init(
             {
                 use_sqlite = true;
             }
+            else if (boost::iequals(get(rdb_section, "backend"), "rwdb"))
+            {
+                use_rwdb = true;
+            }
+            else if (boost::iequals(get(rdb_section, "backend"), "flatmap"))
+            {
+                use_flatmap = true;
+            }
             else
             {
                 Throw<std::runtime_error>(
@@ -73,6 +85,14 @@ RelationalDatabase::init(
     {
         return getPostgresDatabase(app, config, jobQueue);
     }
+    else if (use_rwdb)
+    {
+        return getRWDBDatabase(app, config, jobQueue);
+    }
+    else if (use_flatmap)
+    {
+        return getFlatmapDatabase(app, config, jobQueue);
+    }
 
     return std::unique_ptr<RelationalDatabase>();
 }

src/ripple/app/rdb/impl/UnitaryShard.cpp

@@ -103,22 +103,24 @@ updateLedgerDBs(
 
             for (auto const& item : ledger->txs)
             {
-                if (stop)
+                if (stop.load(std::memory_order_relaxed))
                     return false;
 
-                auto const txID{item.first->getTransactionID()};
-                auto const sTxID{to_string(txID)};
-                auto const txMeta{std::make_shared<TxMeta>(
-                    txID, ledger->seq(), *item.second)};
+                TxMeta const txMeta{
+                    item.first->getTransactionID(),
+                    ledger->seq(),
+                    *item.second};
+
+                auto const sTxID = to_string(txMeta.getTxID());
 
                 session << "DELETE FROM AccountTransactions "
                            "WHERE TransID = :txID;",
                     soci::use(sTxID);
 
-                auto const& accounts = txMeta->getAffectedAccounts();
+                auto const& accounts = txMeta.getAffectedAccounts();
                 if (!accounts.empty())
                 {
-                    auto const sTxnSeq{std::to_string(txMeta->getIndex())};
+                    auto const sTxnSeq{std::to_string(txMeta.getIndex())};
                     auto const s{boost::str(
                         boost::format("('%s','%s',%s,%s)") % sTxID % "%s" %
                         sSeq % sTxnSeq)};

src/ripple/app/reporting/ETLSource.cpp

@@ -194,8 +194,14 @@ ETLSource::onResolve(
     {
         boost::beast::get_lowest_layer(*ws_).expires_after(
             std::chrono::seconds(30));
+
+        // Use async_connect with the entire results
         boost::beast::get_lowest_layer(*ws_).async_connect(
-            results, [this](auto ec, auto ep) { onConnect(ec, ep); });
+            results,
+            [this](
+                boost::beast::error_code ec,
+                boost::asio::ip::tcp::resolver::results_type::endpoint_type
+                    ep) { onConnect(ec, ep); });
     }
 }
 

src/ripple/app/tx/impl/Change.cpp

@@ -458,6 +458,13 @@ Change::activateXahauGenesis()
     bool const isTest =
         (ctx_.tx.getFlags() & tfTestSuite) && ctx_.app.config().standalone();
 
+    // RH NOTE: we'll only configure xahau governance structure on networks that
+    // begin with 2133... so production xahau: 21337 and its testnet 21338
+    // with 21330-21336 and 21339 also valid and reserved for dev nets etc.
+    // all other Network IDs will be conventionally configured.
+    if ((ctx_.app.config().NETWORK_ID / 10) != 2133 && !isTest)
+        return;
+
     auto [ng_entries, l1_entries, l2_entries, gov_params] =
         normalizeXahauGenesis(
             isTest ? TestNonGovernanceDistribution : NonGovernanceDistribution,
@@ -580,7 +587,8 @@ Change::activateXahauGenesis()
                 wasmBytes,  // wasm to verify
                 loggerStream,
                 "rHb9CJAWyB4rj91VRWn96DkukG4bwdtyTh",
-                ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0);
+                (ctx_.view().rules().enabled(featureHooksUpdate1) ? 1 : 0) +
+                    (ctx_.view().rules().enabled(fix20250131) ? 2 : 0));
 
             if (!result)
             {

src/ripple/app/tx/impl/GenesisMint.cpp

@@ -23,6 +23,7 @@
 #include <ripple/ledger/View.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/st.h>
 
 namespace ripple {
 

src/ripple/app/tx/impl/Import.cpp

@@ -24,12 +24,15 @@
 #include <ripple/basics/base64.h>
 #include <ripple/json/json_reader.h>
 #include <ripple/json/json_value.h>
+#include <ripple/json/to_string.h>
 #include <ripple/ledger/View.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/Import.h>
 #include <ripple/protocol/Indexes.h>
 #include <ripple/protocol/PublicKey.h>
 #include <ripple/protocol/STTx.h>
+#include <ripple/protocol/STValidation.h>
+#include <ripple/protocol/st.h>
 #include <algorithm>
 #include <charconv>
 #include <iostream>
@@ -886,6 +889,45 @@ Import::preclaim(PreclaimContext const& ctx)
     }
 
     auto const& sle = ctx.view.read(keylet::account(ctx.tx[sfAccount]));
+
+    auto const tt = stpTrans->getTxnType();
+    if ((tt == ttSIGNER_LIST_SET || tt == ttREGULAR_KEY_SET) &&
+        ctx.view.rules().enabled(fixReduceImport) && sle)
+    {
+        // blackhole check
+        do
+        {
+            // if master key is not set then it is not blackholed
+            if (!(sle->getFlags() & lsfDisableMaster))
+                break;
+
+            // if a regular key is set then it must be acc 0, 1, or 2 otherwise
+            // not blackholed
+            if (sle->isFieldPresent(sfRegularKey))
+            {
+                AccountID rk = sle->getAccountID(sfRegularKey);
+                static const AccountID ACCOUNT_ZERO(0);
+                static const AccountID ACCOUNT_ONE(1);
+                static const AccountID ACCOUNT_TWO(2);
+
+                if (rk != ACCOUNT_ZERO && rk != ACCOUNT_ONE &&
+                    rk != ACCOUNT_TWO)
+                    break;
+            }
+
+            // if a signer list is set then it's not blackholed
+            auto const signerListKeylet = keylet::signers(ctx.tx[sfAccount]);
+            if (ctx.view.exists(signerListKeylet))
+                break;
+
+            // execution to here means it's blackholed
+            JLOG(ctx.j.warn())
+                << "Import: during preclaim target account is blackholed "
+                << ctx.tx[sfAccount] << ", bailing.";
+            return tefIMPORT_BLACKHOLED;
+        } while (0);
+    }
+
     if (sle && sle->isFieldPresent(sfImportSequence))
     {
         uint32_t sleImportSequence = sle->getFieldU32(sfImportSequence);

src/ripple/app/tx/impl/InvariantCheck.h

@@ -319,6 +319,17 @@ public:
         beast::Journal const&);
 };
 
+/**
+ * @brief Invariant: Validates several invariants for NFToken pages.
+ *
+ * The following checks are made:
+ *  - The page is correctly associated with the owner.
+ *  - The page is correctly ordered between the next and previous links.
+ *  - The page contains at least one and no more than 32 NFTokens.
+ *  - The NFTokens on this page do not belong on a lower or higher page.
+ *  - The NFTokens are correctly sorted on the page.
+ *  - Each URI, if present, is not empty.
+ */
 class ValidNFTokenPage
 {
     bool badEntry_ = false;
@@ -343,6 +354,19 @@ public:
         beast::Journal const&);
 };
 
+/**
+ * @brief Invariant: Validates counts of NFTokens after all transaction types.
+ *
+ * The following checks are made:
+ *  - The number of minted or burned NFTokens can only be changed by
+ *    NFTokenMint or NFTokenBurn transactions.
+ *  - A successful NFTokenMint must increase the number of NFTokens.
+ *  - A failed NFTokenMint must not change the number of minted NFTokens.
+ *  - An NFTokenMint transaction cannot change the number of burned NFTokens.
+ *  - A successful NFTokenBurn must increase the number of burned NFTokens.
+ *  - A failed NFTokenBurn must not change the number of burned NFTokens.
+ *  - An NFTokenBurn transaction cannot change the number of minted NFTokens.
+ */
 class NFTokenCountTracking
 {
     std::uint32_t beforeMintedTotal = 0;

src/ripple/app/tx/impl/Invoke.cpp

@@ -22,6 +22,7 @@
 #include <ripple/ledger/View.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/st.h>
 
 namespace ripple {
 

src/ripple/app/tx/impl/Remit.cpp

@@ -23,6 +23,9 @@
 #include <ripple/ledger/View.h>
 #include <ripple/protocol/Feature.h>
 #include <ripple/protocol/Indexes.h>
+#include <ripple/protocol/TxFlags.h>
+#include <ripple/protocol/st.h>
+
 namespace ripple {
 
 TxConsequences
@@ -69,6 +72,16 @@ Remit::preflight(PreflightContext const& ctx)
         return temREDUNDANT;
     }
 
+    if (ctx.rules.enabled(fix20250131))
+    {
+        if (!dstID || dstID == noAccount())
+        {
+            JLOG(ctx.j.warn())
+                << "Malformed transaction: Remit to invalid account.";
+            return temMALFORMED;
+        }
+    }
+
     if (ctx.tx.isFieldPresent(sfInform))
     {
         AccountID const infID = ctx.tx.getAccountID(sfInform);

src/ripple/app/tx/impl/SetHook.cpp

@@ -479,7 +479,8 @@ SetHook::validateHookSetEntry(SetHookCtx& ctx, STObject const& hookSetObj)
                     hook,  // wasm to verify
                     logger,
                     hsacc,
-                    ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0);
+                    (ctx.rules.enabled(featureHooksUpdate1) ? 1 : 0) +
+                        (ctx.rules.enabled(fix20250131) ? 2 : 0));
 
                 if (ctx.j.trace())
                 {

src/ripple/app/tx/impl/SetSignerList.cpp

@@ -153,8 +153,10 @@ SetSignerList::preCompute()
 // NOTE: This way of computing the OwnerCount associated with a SignerList
 // is valid until the featureMultiSignReserve amendment passes.  Once it
 // passes then just 1 OwnerCount is associated with a SignerList.
-static int
-signerCountBasedOwnerCountDelta(std::size_t entryCount, Rules const& rules)
+int
+SetSignerList::signerCountBasedOwnerCountDelta(
+    std::size_t entryCount,
+    Rules const& rules)
 {
     // We always compute the full change in OwnerCount, taking into account:
     //  o The fact that we're adding/removing a SignerList and
@@ -176,8 +178,8 @@ signerCountBasedOwnerCountDelta(std::size_t entryCount, Rules const& rules)
     return 2 + static_cast<int>(entryCount);
 }
 
-static TER
-removeSignersFromLedger(
+TER
+SetSignerList::removeSignersFromLedger(
     Application& app,
     ApplyView& view,
     Keylet const& accountKeylet,

src/ripple/app/tx/impl/SetSignerList.h

@@ -71,6 +71,18 @@ public:
         AccountID const& account,
         beast::Journal j);
 
+    static TER
+    removeSignersFromLedger(
+        Application& app,
+        ApplyView& view,
+        Keylet const& accountKeylet,
+        Keylet const& ownerDirKeylet,
+        Keylet const& signerListKeylet,
+        beast::Journal j);
+
+    static int
+    signerCountBasedOwnerCountDelta(std::size_t entryCount, Rules const& rules);
+
 private:
     static void
     writeSignersToSLE(

src/ripple/app/tx/impl/Transactor.cpp

@@ -1270,10 +1270,18 @@ Transactor::executeHookChain(
                 if (results.back().exitType == hook_api::ExitType::WASM_ERROR)
                 {
                     JLOG(j_.warn()) << "HookError[" << account << "-"
-                                    << ctx_.tx.getAccountID(sfAccount) << "]: "
+                                    << ctx_.tx.getAccountID(sfAccount)
                                     << "]: Execution failure (graceful) "
                                     << "HookHash: " << hookHash;
                 }
+                if (results.back().exitType == hook_api::ExitType::UNSET)
+                {
+                    JLOG(j_.warn())
+                        << "HookError[" << account << "-"
+                        << ctx_.tx.getAccountID(sfAccount)
+                        << "]: Execution failure (no exit type specified) "
+                        << "HookHash: " << hookHash;
+                }
                 return tecHOOK_REJECTED;
             }
 
@@ -1298,7 +1306,7 @@ Transactor::executeHookChain(
         {
             JLOG(j_.warn())
                 << "HookError[" << account << "-"
-                << ctx_.tx.getAccountID(sfAccount) << "]: "
+                << ctx_.tx.getAccountID(sfAccount)
                 << "]: Execution failure (exceptional) "
                 << "Exception: " << e.what() << " HookHash: " << hookHash;
 
@@ -1426,13 +1434,13 @@ Transactor::doHookCallback(
                 finalizeHookResult(callbackResult, ctx_, success);
 
             JLOG(j_.trace()) << "HookInfo[" << callbackAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
-                             << "Callback finalizeHookResult = " << result;
+                             << ctx_.tx.getAccountID(sfAccount)
+                             << "]: Callback finalizeHookResult = " << result;
         }
         catch (std::exception& e)
         {
             JLOG(j_.fatal()) << "HookError[" << callbackAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
+                             << ctx_.tx.getAccountID(sfAccount)
                              << "]: Callback failure " << e.what();
         }
     }
@@ -1678,13 +1686,13 @@ Transactor::doAgainAsWeak(
             results.push_back(aawResult);
 
             JLOG(j_.trace()) << "HookInfo[" << hookAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
-                             << " aaw Hook ExitCode = " << aawResult.exitCode;
+                             << ctx_.tx.getAccountID(sfAccount)
+                             << "]: aaw Hook ExitCode = " << aawResult.exitCode;
         }
         catch (std::exception& e)
         {
             JLOG(j_.fatal()) << "HookError[" << hookAccountID << "-"
-                             << ctx_.tx.getAccountID(sfAccount) << "]: "
+                             << ctx_.tx.getAccountID(sfAccount)
                              << "]: aaw failure " << e.what();
         }
     }
@@ -1923,6 +1931,7 @@ Transactor::operator()()
         uint32_t lgrCur = view().seq();
 
         bool const has240819 = view().rules().enabled(fix240819);
+        bool const has240911 = view().rules().enabled(fix240911);
 
         auto const& sfRewardFields =
             *(ripple::SField::knownCodeToField.at(917511 - has240819));
@@ -1971,7 +1980,11 @@ Transactor::operator()()
             uint32_t lgrElapsed = lgrCur - lgrLast;
 
             // overflow safety
-            if (lgrElapsed > lgrCur || lgrElapsed > lgrLast || lgrElapsed == 0)
+            if (!has240911 &&
+                (lgrElapsed > lgrCur || lgrElapsed > lgrLast ||
+                 lgrElapsed == 0))
+                continue;
+            if (has240911 && (lgrElapsed > lgrCur || lgrElapsed == 0))
                 continue;
 
             uint64_t accum = sle->getFieldU64(sfRewardAccumulator);

src/ripple/app/tx/impl/applySteps.cpp

@@ -23,6 +23,7 @@
 #include <ripple/app/tx/impl/CancelOffer.h>
 #include <ripple/app/tx/impl/CashCheck.h>
 #include <ripple/app/tx/impl/Change.h>
+#include <ripple/app/tx/impl/ClaimReward.h>
 #include <ripple/app/tx/impl/CreateCheck.h>
 #include <ripple/app/tx/impl/CreateOffer.h>
 #include <ripple/app/tx/impl/CreateTicket.h>
@@ -39,12 +40,14 @@
 #include <ripple/app/tx/impl/NFTokenMint.h>
 #include <ripple/app/tx/impl/PayChan.h>
 #include <ripple/app/tx/impl/Payment.h>
+#include <ripple/app/tx/impl/Remit.h>
 #include <ripple/app/tx/impl/SetAccount.h>
 #include <ripple/app/tx/impl/SetHook.h>
 #include <ripple/app/tx/impl/SetRegularKey.h>
 #include <ripple/app/tx/impl/SetSignerList.h>
 #include <ripple/app/tx/impl/SetTrust.h>
 #include <ripple/app/tx/impl/URIToken.h>
+#include <ripple/app/tx/impl/XahauGenesis.h>
 
 namespace ripple {
 

src/ripple/basics/IOUAmount.h

@@ -186,7 +186,14 @@ mulRatio(
     std::uint32_t den,
     bool roundUp);
 
-extern LocalValue<bool> stNumberSwitchover;
+// Since many uses of the number class do not have access to a ledger,
+// getSTNumberSwitchover needs to be globally accessible.
+
+bool
+getSTNumberSwitchover();
+
+void
+setSTNumberSwitchover(bool v);
 
 /** RAII class to set and restore the Number switchover.
  */
@@ -198,16 +205,16 @@ class NumberSO
 public:
     ~NumberSO()
     {
-        *stNumberSwitchover = saved_;
+        setSTNumberSwitchover(saved_);
     }
 
     NumberSO(NumberSO const&) = delete;
     NumberSO&
     operator=(NumberSO const&) = delete;
 
-    explicit NumberSO(bool v) : saved_(*stNumberSwitchover)
+    explicit NumberSO(bool v) : saved_(getSTNumberSwitchover())
     {
-        *stNumberSwitchover = v;
+        setSTNumberSwitchover(v);
     }
 };
 

src/ripple/basics/Log.h

@@ -24,6 +24,7 @@
 #include <ripple/beast/utility/Journal.h>
 #include <boost/beast/core/string.hpp>
 #include <boost/filesystem.hpp>
+#include <fstream>
 #include <map>
 #include <memory>
 #include <mutex>

src/ripple/basics/SlabAllocator.h

@@ -0,0 +1,432 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of rippled: https://github.com/ripple/rippled
+    Copyright 2022, Nikolaos D. Bougalis <nikb@bougalis.net>
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose  with  or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL ,  DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#ifndef RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED
+#define RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED
+
+#include <ripple/beast/type_name.h>
+
+#include <boost/align.hpp>
+#include <boost/container/static_vector.hpp>
+#include <boost/predef.h>
+
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <cstdint>
+#include <cstring>
+#include <mutex>
+
+#if BOOST_OS_LINUX
+#include <sys/mman.h>
+#endif
+
+namespace ripple {
+
+template <typename Type>
+class SlabAllocator
+{
+    static_assert(
+        sizeof(Type) >= sizeof(std::uint8_t*),
+        "SlabAllocator: the requested object must be larger than a pointer.");
+
+    static_assert(alignof(Type) == 8 || alignof(Type) == 4);
+
+    /** A block of memory that is owned by a slab allocator */
+    struct SlabBlock
+    {
+        // A mutex to protect the freelist for this block:
+        std::mutex m_;
+
+        // A linked list of appropriately sized free buffers:
+        std::uint8_t* l_ = nullptr;
+
+        // The next memory block
+        SlabBlock* next_;
+
+        // The underlying memory block:
+        std::uint8_t const* const p_ = nullptr;
+
+        // The extent of the underlying memory block:
+        std::size_t const size_;
+
+        SlabBlock(
+            SlabBlock* next,
+            std::uint8_t* data,
+            std::size_t size,
+            std::size_t item)
+            : next_(next), p_(data), size_(size)
+        {
+            // We don't need to grab the mutex here, since we're the only
+            // ones with access at this moment.
+
+            while (data + item <= p_ + size_)
+            {
+                // Use memcpy to avoid unaligned UB
+                // (will optimize to equivalent code)
+                std::memcpy(data, &l_, sizeof(std::uint8_t*));
+                l_ = data;
+                data += item;
+            }
+        }
+
+        ~SlabBlock()
+        {
+            // Calling this destructor will release the allocated memory but
+            // will not properly destroy any objects that are constructed in
+            // the block itself.
+        }
+
+        SlabBlock(SlabBlock const& other) = delete;
+        SlabBlock&
+        operator=(SlabBlock const& other) = delete;
+
+        SlabBlock(SlabBlock&& other) = delete;
+        SlabBlock&
+        operator=(SlabBlock&& other) = delete;
+
+        /** Determines whether the given pointer belongs to this allocator */
+        bool
+        own(std::uint8_t const* p) const noexcept
+        {
+            return (p >= p_) && (p < p_ + size_);
+        }
+
+        std::uint8_t*
+        allocate() noexcept
+        {
+            std::uint8_t* ret;
+
+            {
+                std::lock_guard l(m_);
+
+                ret = l_;
+
+                if (ret)
+                {
+                    // Use memcpy to avoid unaligned UB
+                    // (will optimize to equivalent code)
+                    std::memcpy(&l_, ret, sizeof(std::uint8_t*));
+                }
+            }
+
+            return ret;
+        }
+
+        /** Return an item to this allocator's freelist.
+
+            @param ptr The pointer to the chunk of memory being deallocated.
+
+            @note This is a dangerous, private interface; the item being
+                  returned should belong to this allocator. Debug builds
+                  will check and assert if this is not the case. Release
+                  builds will not.
+         */
+        void
+        deallocate(std::uint8_t* ptr) noexcept
+        {
+            assert(own(ptr));
+
+            std::lock_guard l(m_);
+
+            // Use memcpy to avoid unaligned UB
+            // (will optimize to equivalent code)
+            std::memcpy(ptr, &l_, sizeof(std::uint8_t*));
+            l_ = ptr;
+        }
+    };
+
+private:
+    // A linked list of slabs
+    std::atomic<SlabBlock*> slabs_ = nullptr;
+
+    // The alignment requirements of the item we're allocating:
+    std::size_t const itemAlignment_;
+
+    // The size of an item, including the extra bytes requested and
+    // any padding needed for alignment purposes:
+    std::size_t const itemSize_;
+
+    // The size of each individual slab:
+    std::size_t const slabSize_;
+
+public:
+    /** Constructs a slab allocator able to allocate objects of a fixed size
+
+        @param count the number of items the slab allocator can allocate; note
+                     that a count of 0 is valid and means that the allocator
+                     is, effectively, disabled. This can be very useful in some
+                     contexts (e.g. when mimimal memory usage is needed) and
+                     allows for graceful failure.
+     */
+    constexpr explicit SlabAllocator(
+        std::size_t extra,
+        std::size_t alloc = 0,
+        std::size_t align = 0)
+        : itemAlignment_(align ? align : alignof(Type))
+        , itemSize_(
+              boost::alignment::align_up(sizeof(Type) + extra, itemAlignment_))
+        , slabSize_(alloc)
+    {
+        assert((itemAlignment_ & (itemAlignment_ - 1)) == 0);
+    }
+
+    SlabAllocator(SlabAllocator const& other) = delete;
+    SlabAllocator&
+    operator=(SlabAllocator const& other) = delete;
+
+    SlabAllocator(SlabAllocator&& other) = delete;
+    SlabAllocator&
+    operator=(SlabAllocator&& other) = delete;
+
+    ~SlabAllocator()
+    {
+        // FIXME: We can't destroy the memory blocks we've allocated, because
+        //        we can't be sure that they are not being used. Cleaning the
+        //        shutdown process up could make this possible.
+    }
+
+    /** Returns the size of the memory block this allocator returns. */
+    constexpr std::size_t
+    size() const noexcept
+    {
+        return itemSize_;
+    }
+
+    /** Returns a suitably aligned pointer, if one is available.
+
+        @return a pointer to a block of memory from the allocator, or
+                nullptr if the allocator can't satisfy this request.
+     */
+    std::uint8_t*
+    allocate() noexcept
+    {
+        auto slab = slabs_.load();
+
+        while (slab != nullptr)
+        {
+            if (auto ret = slab->allocate())
+                return ret;
+
+            slab = slab->next_;
+        }
+
+        // No slab can satisfy our request, so we attempt to allocate a new
+        // one here:
+        std::size_t size = slabSize_;
+
+        // We want to allocate the memory at a 2 MiB boundary, to make it
+        // possible to use hugepage mappings on Linux:
+        auto buf =
+            boost::alignment::aligned_alloc(megabytes(std::size_t(2)), size);
+
+        // clang-format off
+        if (!buf) [[unlikely]]
+            return nullptr;
+            // clang-format on
+
+#if BOOST_OS_LINUX
+        // When allocating large blocks, attempt to leverage Linux's
+        // transparent hugepage support. It is unclear and difficult
+        // to accurately determine if doing this impacts performance
+        // enough to justify using platform-specific tricks.
+        if (size >= megabytes(std::size_t(4)))
+            madvise(buf, size, MADV_HUGEPAGE);
+#endif
+
+        // We need to carve out a bit of memory for the slab header
+        // and then align the rest appropriately:
+        auto slabData = reinterpret_cast<void*>(
+            reinterpret_cast<std::uint8_t*>(buf) + sizeof(SlabBlock));
+        auto slabSize = size - sizeof(SlabBlock);
+
+        // This operation is essentially guaranteed not to fail but
+        // let's be careful anyways.
+        if (!boost::alignment::align(
+                itemAlignment_, itemSize_, slabData, slabSize))
+        {
+            boost::alignment::aligned_free(buf);
+            return nullptr;
+        }
+
+        slab = new (buf) SlabBlock(
+            slabs_.load(),
+            reinterpret_cast<std::uint8_t*>(slabData),
+            slabSize,
+            itemSize_);
+
+        // Link the new slab
+        while (!slabs_.compare_exchange_weak(
+            slab->next_,
+            slab,
+            std::memory_order_release,
+            std::memory_order_relaxed))
+        {
+            ;  // Nothing to do
+        }
+
+        return slab->allocate();
+    }
+
+    /** Returns the memory block to the allocator.
+
+        @param ptr A pointer to a memory block.
+        @param size If non-zero, a hint as to the size of the block.
+        @return true if this memory block belonged to the allocator and has
+                     been released; false otherwise.
+     */
+    bool
+    deallocate(std::uint8_t* ptr) noexcept
+    {
+        assert(ptr);
+
+        for (auto slab = slabs_.load(); slab != nullptr; slab = slab->next_)
+        {
+            if (slab->own(ptr))
+            {
+                slab->deallocate(ptr);
+                return true;
+            }
+        }
+
+        return false;
+    }
+};
+
+/** A collection of slab allocators of various sizes for a given type. */
+template <typename Type>
+class SlabAllocatorSet
+{
+private:
+    // The list of allocators that belong to this set
+    boost::container::static_vector<SlabAllocator<Type>, 64> allocators_;
+
+    std::size_t maxSize_ = 0;
+
+public:
+    class SlabConfig
+    {
+        friend class SlabAllocatorSet;
+
+    private:
+        std::size_t extra;
+        std::size_t alloc;
+        std::size_t align;
+
+    public:
+        constexpr SlabConfig(
+            std::size_t extra_,
+            std::size_t alloc_ = 0,
+            std::size_t align_ = alignof(Type))
+            : extra(extra_), alloc(alloc_), align(align_)
+        {
+        }
+    };
+
+    constexpr SlabAllocatorSet(std::vector<SlabConfig> cfg)
+    {
+        // Ensure that the specified allocators are sorted from smallest to
+        // largest by size:
+        std::sort(
+            std::begin(cfg),
+            std::end(cfg),
+            [](SlabConfig const& a, SlabConfig const& b) {
+                return a.extra < b.extra;
+            });
+
+        // We should never have two slabs of the same size
+        if (std::adjacent_find(
+                std::begin(cfg),
+                std::end(cfg),
+                [](SlabConfig const& a, SlabConfig const& b) {
+                    return a.extra == b.extra;
+                }) != cfg.end())
+        {
+            throw std::runtime_error(
+                "SlabAllocatorSet<" + beast::type_name<Type>() +
+                ">: duplicate slab size");
+        }
+
+        for (auto const& c : cfg)
+        {
+            auto& a = allocators_.emplace_back(c.extra, c.alloc, c.align);
+
+            if (a.size() > maxSize_)
+                maxSize_ = a.size();
+        }
+    }
+
+    SlabAllocatorSet(SlabAllocatorSet const& other) = delete;
+    SlabAllocatorSet&
+    operator=(SlabAllocatorSet const& other) = delete;
+
+    SlabAllocatorSet(SlabAllocatorSet&& other) = delete;
+    SlabAllocatorSet&
+    operator=(SlabAllocatorSet&& other) = delete;
+
+    ~SlabAllocatorSet()
+    {
+    }
+
+    /** Returns a suitably aligned pointer, if one is available.
+
+        @param extra The number of extra bytes, above and beyond the size of
+                     the object, that should be returned by the allocator.
+
+        @return a pointer to a block of memory, or nullptr if the allocator
+                can't satisfy this request.
+     */
+    std::uint8_t*
+    allocate(std::size_t extra) noexcept
+    {
+        if (auto const size = sizeof(Type) + extra; size <= maxSize_)
+        {
+            for (auto& a : allocators_)
+            {
+                if (a.size() >= size)
+                    return a.allocate();
+            }
+        }
+
+        return nullptr;
+    }
+
+    /** Returns the memory block to the allocator.
+
+        @param ptr A pointer to a memory block.
+
+        @return true if this memory block belonged to one of the allocators
+                     in this set and has been released; false otherwise.
+     */
+    bool
+    deallocate(std::uint8_t* ptr) noexcept
+    {
+        for (auto& a : allocators_)
+        {
+            if (a.deallocate(ptr))
+                return true;
+        }
+
+        return false;
+    }
+};
+
+}  // namespace ripple
+
+#endif  // RIPPLE_BASICS_SLABALLOCATOR_H_INCLUDED

src/ripple/basics/StringUtilities.h

@@ -25,7 +25,9 @@
 
 #include <boost/format.hpp>
 #include <boost/utility/string_view.hpp>
+
 #include <array>
+#include <cstdint>
 #include <optional>
 #include <sstream>
 #include <string>

src/ripple/basics/base64.h

@@ -57,6 +57,7 @@
 #ifndef RIPPLE_BASICS_BASE64_H_INCLUDED
 #define RIPPLE_BASICS_BASE64_H_INCLUDED
 
+#include <cstdint>
 #include <string>
 
 namespace ripple {

src/ripple/basics/base_uint.h

@@ -549,6 +549,7 @@ using uint128 = base_uint<128>;
 using uint160 = base_uint<160>;
 using uint256 = base_uint<256>;
 
+#ifdef __APPLE__
 template <std::size_t Bits, class Tag>
 [[nodiscard]] inline constexpr std::strong_ordering
 operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
@@ -560,7 +561,6 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
     //
     // FIXME: use std::lexicographical_compare_three_way once support is
     //        added to MacOS.
-
     auto const ret = std::mismatch(lhs.cbegin(), lhs.cend(), rhs.cbegin());
 
     // a == b
@@ -570,6 +570,19 @@ operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
     return (*ret.first > *ret.second) ? std::strong_ordering::greater
                                       : std::strong_ordering::less;
 }
+#else
+template <std::size_t Bits, class Tag>
+[[nodiscard]] inline constexpr std::strong_ordering
+operator<=>(base_uint<Bits, Tag> const& lhs, base_uint<Bits, Tag> const& rhs)
+{
+    return std::lexicographical_compare_three_way(
+        lhs.cbegin(),
+        lhs.cend(),
+        rhs.cbegin(),
+        rhs.cend(),
+        std::compare_three_way{});
+}
+#endif
 
 template <std::size_t Bits, typename Tag>
 [[nodiscard]] inline constexpr bool

src/ripple/basics/impl/FileUtilities.cpp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <ripple/basics/FileUtilities.h>
+#include <fstream>
 
 namespace ripple {
 
@@ -41,7 +42,7 @@ getFileContents(
         return {};
     }
 
-    ifstream fileStream(fullPath, std::ios::in);
+    std::ifstream fileStream(fullPath.string(), std::ios::in);
 
     if (!fileStream)
     {
@@ -71,7 +72,8 @@ writeFileContents(
     using namespace boost::filesystem;
     using namespace boost::system::errc;
 
-    ofstream fileStream(destPath, std::ios::out | std::ios::trunc);
+    std::ofstream fileStream(
+        destPath.string(), std::ios::out | std::ios::trunc);
 
     if (!fileStream)
     {

src/ripple/basics/impl/IOUAmount.cpp

@@ -27,7 +27,28 @@
 
 namespace ripple {
 
-LocalValue<bool> stNumberSwitchover(true);
+namespace {
+
+// Use a static inside a function to help prevent order-of-initialzation issues
+LocalValue<bool>&
+getStaticSTNumberSwitchover()
+{
+    static LocalValue<bool> r{true};
+    return r;
+}
+}  // namespace
+
+bool
+getSTNumberSwitchover()
+{
+    return *getStaticSTNumberSwitchover();
+}
+
+void
+setSTNumberSwitchover(bool v)
+{
+    *getStaticSTNumberSwitchover() = v;
+}
 
 /* The range for the mantissa when normalized */
 static std::int64_t constexpr minMantissa = 1000000000000000ull;
@@ -51,7 +72,7 @@ IOUAmount::normalize()
         return;
     }
 
-    if (*stNumberSwitchover)
+    if (getSTNumberSwitchover())
     {
         Number const v{mantissa_, exponent_};
         mantissa_ = v.mantissa();
@@ -117,7 +138,7 @@ IOUAmount::operator+=(IOUAmount const& other)
         return *this;
     }
 
-    if (*stNumberSwitchover)
+    if (getSTNumberSwitchover())
     {
         *this = IOUAmount{Number{*this} + Number{other}};
         return *this;

src/ripple/basics/impl/mulDiv.cpp

@@ -30,7 +30,7 @@ mulDiv(std::uint64_t value, std::uint64_t mul, std::uint64_t div)
 {
     using namespace boost::multiprecision;
 
-    uint128_t result;
+    boost::multiprecision::uint128_t result;
     result = multiply(result, value, mul);
 
     result /= div;

src/ripple/basics/impl/partitioned_unordered_map.cpp

@@ -31,7 +31,11 @@ namespace ripple {
 static std::size_t
 extract(uint256 const& key)
 {
-    return *reinterpret_cast<std::size_t const*>(key.data());
+    std::size_t result;
+    // Use memcpy to avoid unaligned UB
+    // (will optimize to equivalent code)
+    std::memcpy(&result, key.data(), sizeof(std::size_t));
+    return result;
 }
 
 static std::size_t

src/ripple/beast/container/detail/aged_ordered_container.h

@@ -145,111 +145,78 @@ private:
     };
 
     // VFALCO TODO This should only be enabled for maps.
-    class pair_value_compare
-        : public beast::detail::empty_base_optimization<Compare>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<value_type, value_type, bool>
-#endif
+    class pair_value_compare : public Compare
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument = value_type;
         using second_argument = value_type;
         using result_type = bool;
-#endif
 
         bool
         operator()(value_type const& lhs, value_type const& rhs) const
         {
-            return this->member()(lhs.first, rhs.first);
+            return Compare::operator()(lhs.first, rhs.first);
         }
 
         pair_value_compare()
         {
         }
 
-        pair_value_compare(pair_value_compare const& other)
-            : beast::detail::empty_base_optimization<Compare>(other)
+        pair_value_compare(pair_value_compare const& other) : Compare(other)
         {
         }
 
     private:
         friend aged_ordered_container;
 
-        pair_value_compare(Compare const& compare)
-            : beast::detail::empty_base_optimization<Compare>(compare)
+        pair_value_compare(Compare const& compare) : Compare(compare)
         {
         }
     };
 
     // Compares value_type against element, used in insert_check
     // VFALCO TODO hoist to remove template argument dependencies
-    class KeyValueCompare
-        : public beast::detail::empty_base_optimization<Compare>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<Key, element, bool>
-#endif
+    class KeyValueCompare : public Compare
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument = Key;
         using second_argument = element;
         using result_type = bool;
-#endif
 
         KeyValueCompare() = default;
 
-        KeyValueCompare(Compare const& compare)
-            : beast::detail::empty_base_optimization<Compare>(compare)
+        KeyValueCompare(Compare const& compare) : Compare(compare)
         {
         }
 
-        // VFALCO NOTE WE might want only to enable these overloads
-        //                if Compare has is_transparent
-#if 0
-        template <class K>
-        bool operator() (K const& k, element const& e) const
-        {
-            return this->member() (k, extract (e.value));
-        }
-
-        template <class K>
-        bool operator() (element const& e, K const& k) const
-        {
-            return this->member() (extract (e.value), k);
-        }
-#endif
-
         bool
         operator()(Key const& k, element const& e) const
         {
-            return this->member()(k, extract(e.value));
+            return Compare::operator()(k, extract(e.value));
         }
 
         bool
         operator()(element const& e, Key const& k) const
         {
-            return this->member()(extract(e.value), k);
+            return Compare::operator()(extract(e.value), k);
         }
 
         bool
         operator()(element const& x, element const& y) const
         {
-            return this->member()(extract(x.value), extract(y.value));
+            return Compare::operator()(extract(x.value), extract(y.value));
         }
 
         Compare&
         compare()
         {
-            return beast::detail::empty_base_optimization<Compare>::member();
+            return *this;
         }
 
         Compare const&
         compare() const
         {
-            return beast::detail::empty_base_optimization<Compare>::member();
+            return *this;
         }
     };
 

src/ripple/beast/container/detail/aged_unordered_container.h

@@ -148,115 +148,84 @@ private:
     };
 
     // VFALCO TODO hoist to remove template argument dependencies
-    class ValueHash : private beast::detail::empty_base_optimization<Hash>
-#ifdef _LIBCPP_VERSION
-        ,
-                      public std::unary_function<element, std::size_t>
-#endif
+    class ValueHash : public Hash
     {
     public:
-#ifndef _LIBCPP_VERSION
         using argument_type = element;
         using result_type = size_t;
-#endif
 
         ValueHash()
         {
         }
 
-        ValueHash(Hash const& h)
-            : beast::detail::empty_base_optimization<Hash>(h)
+        ValueHash(Hash const& h) : Hash(h)
         {
         }
 
         std::size_t
         operator()(element const& e) const
         {
-            return this->member()(extract(e.value));
+            return Hash::operator()(extract(e.value));
         }
 
         Hash&
         hash_function()
         {
-            return this->member();
+            return *this;
         }
 
         Hash const&
         hash_function() const
         {
-            return this->member();
+            return *this;
         }
     };
 
     // Compares value_type against element, used in find/insert_check
     // VFALCO TODO hoist to remove template argument dependencies
-    class KeyValueEqual
-        : private beast::detail::empty_base_optimization<KeyEqual>
-#ifdef _LIBCPP_VERSION
-        ,
-          public std::binary_function<Key, element, bool>
-#endif
+    class KeyValueEqual : public KeyEqual
     {
     public:
-#ifndef _LIBCPP_VERSION
         using first_argument_type = Key;
         using second_argument_type = element;
         using result_type = bool;
-#endif
 
         KeyValueEqual()
         {
         }
 
-        KeyValueEqual(KeyEqual const& keyEqual)
-            : beast::detail::empty_base_optimization<KeyEqual>(keyEqual)
+        KeyValueEqual(KeyEqual const& keyEqual) : KeyEqual(keyEqual)
         {
         }
 
-        // VFALCO NOTE WE might want only to enable these overloads
-        //                if KeyEqual has is_transparent
-#if 0
-        template <class K>
-        bool operator() (K const& k, element const& e) const
-        {
-            return this->member() (k, extract (e.value));
-        }
-
-        template <class K>
-        bool operator() (element const& e, K const& k) const
-        {
-            return this->member() (extract (e.value), k);
-        }
-#endif
-
         bool
         operator()(Key const& k, element const& e) const
         {
-            return this->member()(k, extract(e.value));
+            return KeyEqual::operator()(k, extract(e.value));
         }
 
         bool
         operator()(element const& e, Key const& k) const
         {
-            return this->member()(extract(e.value), k);
+            return KeyEqual::operator()(extract(e.value), k);
         }
 
         bool
         operator()(element const& lhs, element const& rhs) const
         {
-            return this->member()(extract(lhs.value), extract(rhs.value));
+            return KeyEqual::operator()(extract(lhs.value), extract(rhs.value));
         }
 
         KeyEqual&
         key_eq()
         {
-            return this->member();
+            return *this;
         }
 
         KeyEqual const&
         key_eq() const
         {
-            return this->member();
+            return *this;
         }
     };
 

src/ripple/beast/hash/impl/xxhash.cpp

@@ -33,6 +33,8 @@ You can contact the author at :
 
 #include <ripple/beast/hash/impl/xxhash.h>
 
+#include <cstring>
+
 //**************************************
 // Tuning parameters
 //**************************************
@@ -87,7 +89,7 @@ You can contact the author at :
 //**************************************
 // Includes & Memory related functions
 //**************************************
-//#include "xxhash.h"
+// #include "xxhash.h"
 // Modify the local functions below should you wish to use some other memory
 // routines for malloc(), free()
 #include <stdlib.h>
@@ -260,7 +262,13 @@ FORCE_INLINE U64
 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align)
 {
     if (align == XXH_unaligned)
-        return endian == XXH_littleEndian ? A64(ptr) : XXH_swap64(A64(ptr));
+    {
+        // Use memcpy to avoid unaligned UB
+        U64 tmp_aligned;
+        std::memcpy(&tmp_aligned, ptr, sizeof(U64));
+        return endian == XXH_littleEndian ? tmp_aligned
+                                          : XXH_swap64(tmp_aligned);
+    }
     else
         return endian == XXH_littleEndian ? *(U64*)ptr : XXH_swap64(*(U64*)ptr);
 }

src/ripple/consensus/Consensus.h

@@ -1644,7 +1644,7 @@ Consensus<Adaptor>::createDisputes(TxSet_t const& o)
             (inThisSet && result_->txns.find(txId) && !o.find(txId)) ||
             (!inThisSet && !result_->txns.find(txId) && o.find(txId)));
 
-        Tx_t tx = inThisSet ? *result_->txns.find(txId) : *o.find(txId);
+        Tx_t tx = inThisSet ? result_->txns.find(txId) : o.find(txId);
         auto txID = tx.id();
 
         if (result_->disputes.find(txID) != result_->disputes.end())

src/ripple/core/Config.h

@@ -25,6 +25,7 @@
 #include <ripple/basics/base_uint.h>
 #include <ripple/beast/net/IPEndpoint.h>
 #include <ripple/beast/utility/Journal.h>
+#include <ripple/core/ConfigSections.h>
 #include <ripple/protocol/PublicKey.h>
 #include <ripple/protocol/SystemParameters.h>  // VFALCO Breaks levelization
 #include <boost/beast/core/string.hpp>
@@ -154,6 +155,8 @@ public:
     std::map<std::string, PublicKey>
         IMPORT_VL_KEYS;  // hex string -> class PublicKey (for caching purposes)
 
+    std::vector<std::string> DATAGRAM_MONITOR;
+
     enum StartUpType {
         FRESH,
         NORMAL,
@@ -240,7 +243,7 @@ public:
     bool LEDGER_REPLAY = false;
 
     // Work queue limits
-    int MAX_TRANSACTIONS = 250;
+    int MAX_TRANSACTIONS = 1000;
     static constexpr int MAX_JOB_QUEUE_TX = 1000;
     static constexpr int MIN_JOB_QUEUE_TX = 100;
 
@@ -350,6 +353,21 @@ public:
     {
         return RUN_REPORTING;
     }
+    bool
+    mem_backend() const
+    {
+        static bool const isMem =
+            (!section(SECTION_RELATIONAL_DB).empty() &&
+             boost::beast::iequals(
+                 get(section(SECTION_RELATIONAL_DB), "backend"), "rwdb")) ||
+            (!section("node_db").empty() &&
+             (boost::beast::iequals(get(section("node_db"), "type"), "rwdb") ||
+              boost::beast::iequals(
+                  get(section("node_db"), "type"), "flatmap")));
+        // RHNOTE: memory type is not selected for here because it breaks
+        // tests
+        return isMem;
+    }
 
     bool
     useTxTables() const

src/ripple/core/ConfigSections.h

@@ -101,6 +101,7 @@ struct ConfigSection
 #define SECTION_SWEEP_INTERVAL "sweep_interval"
 #define SECTION_NETWORK_ID "network_id"
 #define SECTION_IMPORT_VL_KEYS "import_vl_keys"
+#define SECTION_DATAGRAM_MONITOR "datagram_monitor"
 
 }  // namespace ripple
 

src/ripple/core/impl/Config.cpp

@@ -68,10 +68,8 @@ namespace detail {
 [[nodiscard]] std::uint64_t
 getMemorySize()
 {
-    struct sysinfo si;
-
-    if (sysinfo(&si) == 0)
-        return static_cast<std::uint64_t>(si.totalram);
+    if (struct sysinfo si; sysinfo(&si) == 0)
+        return static_cast<std::uint64_t>(si.totalram) * si.mem_unit;
 
     return 0;
 }
@@ -265,7 +263,8 @@ getEnvVar(char const* name)
 }
 
 Config::Config()
-    : j_(beast::Journal::getNullSink()), ramSize_(detail::getMemorySize())
+    : j_(beast::Journal::getNullSink())
+    , ramSize_(detail::getMemorySize() / (1024 * 1024 * 1024))
 {
 }
 
@@ -282,6 +281,9 @@ Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
     // RAM and CPU resources. We default to "tiny" for standalone mode.
     if (!bStandalone)
     {
+        NODE_SIZE = 4;
+        return;
+
         // First, check against 'minimum' RAM requirements per node size:
         auto const& threshold =
             sizedItems[std::underlying_type_t<SizedItem>(SizedItem::ramSizeGB)];
@@ -290,22 +292,18 @@ Config::setupControl(bool bQuiet, bool bSilent, bool bStandalone)
             threshold.second.begin(),
             threshold.second.end(),
             [this](std::size_t limit) {
-                return (ramSize_ / (1024 * 1024 * 1024)) < limit;
+                return (limit == 0) || (ramSize_ < limit);
             });
 
+        assert(ns != threshold.second.end());
+
         if (ns != threshold.second.end())
             NODE_SIZE = std::distance(threshold.second.begin(), ns);
 
         // Adjust the size based on the number of hardware threads of
         // execution available to us:
-        if (auto const hc = std::thread::hardware_concurrency())
-        {
-            if (hc == 1)
-                NODE_SIZE = 0;
-
-            if (hc < 4)
-                NODE_SIZE = std::min<std::size_t>(NODE_SIZE, 1);
-        }
+        if (auto const hc = std::thread::hardware_concurrency(); hc != 0)
+            NODE_SIZE = std::min<std::size_t>(hc / 2, NODE_SIZE);
     }
 
     assert(NODE_SIZE <= 4);
@@ -470,26 +468,24 @@ Config::loadFromString(std::string const& fileContents)
         SNTP_SERVERS = *s;
 
     // if the user has specified ip:port then replace : with a space.
-    {
-        auto replaceColons = [](std::vector<std::string>& strVec) {
-            const static std::regex e(":([0-9]+)$");
-            for (auto& line : strVec)
-            {
-                // skip anything that might be an ipv6 address
-                if (std::count(line.begin(), line.end(), ':') != 1)
-                    continue;
+    auto replaceColons = [](std::vector<std::string>& strVec) {
+        const static std::regex e(":([0-9]+)$");
+        for (auto& line : strVec)
+        {
+            // skip anything that might be an ipv6 address
+            if (std::count(line.begin(), line.end(), ':') != 1)
+                continue;
 
-                std::string result = std::regex_replace(line, e, " $1");
-                // sanity check the result of the replace, should be same length
-                // as input
-                if (result.size() == line.size())
-                    line = result;
-            }
-        };
+            std::string result = std::regex_replace(line, e, " $1");
+            // sanity check the result of the replace, should be same length
+            // as input
+            if (result.size() == line.size())
+                line = result;
+        }
+    };
 
-        replaceColons(IPS_FIXED);
-        replaceColons(IPS);
-    }
+    replaceColons(IPS_FIXED);
+    replaceColons(IPS);
 
     {
         std::string dbPath;
@@ -514,6 +510,12 @@ Config::loadFromString(std::string const& fileContents)
             NETWORK_ID = beast::lexicalCastThrow<uint32_t>(strTemp);
     }
 
+    if (auto s = getIniFileSection(secConfig, SECTION_DATAGRAM_MONITOR))
+    {
+        DATAGRAM_MONITOR = *s;
+        replaceColons(DATAGRAM_MONITOR);
+    }
+
     if (getSingleSection(secConfig, SECTION_PEER_PRIVATE, strTemp, j_))
         PEER_PRIVATE = beast::lexicalCastThrow<bool>(strTemp);
 

src/ripple/json/impl/json_reader.cpp

@@ -19,8 +19,10 @@
 
 #include <ripple/basics/contract.h>
 #include <ripple/json/json_reader.h>
+
 #include <algorithm>
 #include <cctype>
+#include <cstdint>
 #include <istream>
 #include <string>
 

src/ripple/ledger/impl/ApplyView.cpp

@@ -91,8 +91,10 @@ ApplyView::dirAdd(
         return page;
     }
 
+    bool const capped = !rules().enabled(fixPageCap);
+
     // Check whether we're out of pages.
-    if (++page >= dirNodeMaxPages)
+    if (++page >= dirNodeMaxPages && capped)
         return std::nullopt;
 
     // We are about to create a new node; we'll link it to

src/ripple/ledger/impl/ApplyViewBase.cpp

@@ -19,6 +19,7 @@
 
 #include <ripple/basics/contract.h>
 #include <ripple/ledger/detail/ApplyViewBase.h>
+#include <ripple/protocol/STAccount.h>
 
 namespace ripple {
 namespace detail {

src/ripple/net/ShardDownloader.md

@@ -195,7 +195,7 @@ three database entries upon completion.
 Since downloads execute serially by design, the entries in this table always
 correspond to the contents of a single file.
 
-| Bytes  | Size       | Part |
+| Bytes  | size       | Part |
 |:------:|:----------:|:----:|
 | 0x...  | 2147483647 | 0    |
 | 0x...  | 2147483647 | 1    |

src/ripple/net/impl/DatabaseBody.ipp

@@ -18,6 +18,7 @@
 //==============================================================================
 
 #include <ripple/app/rdb/Download.h>
+#include <fstream>
 
 namespace ripple {
 

src/ripple/net/impl/RPCCall.cpp

@@ -823,11 +823,9 @@ private:
         return jvRequest;
     }
 
-    // owner_info <account>|<account_public_key> [strict]
-    // owner_info <seed>|<pass_phrase>|<key> [<ledger>] [strict]
-    // account_info <account>|<account_public_key> [strict]
-    // account_info <seed>|<pass_phrase>|<key> [<ledger>] [strict]
-    // account_offers <account>|<account_public_key> [<ledger>] [strict]
+    // owner_info <account>
+    // account_info <account> [<ledger>]
+    // account_offers <account> [<ledger>]
     Json::Value
     parseAccountItems(Json::Value const& jvParams)
     {
@@ -1050,10 +1048,7 @@ private:
             // Parameters 0 and 1 are accounts
             if (i < 2)
             {
-                if (parseBase58<PublicKey>(
-                        TokenType::AccountPublic, strParam) ||
-                    parseBase58<AccountID>(strParam) ||
-                    parseGenericSeed(strParam))
+                if (parseBase58<AccountID>(strParam))
                 {
                     jvRequest[accFields[i]] = std::move(strParam);
                 }
@@ -1079,16 +1074,8 @@ private:
     {
         std::string strIdent = jvParams[0u].asString();
         unsigned int iCursor = jvParams.size();
-        bool bStrict = false;
 
-        if (iCursor >= 2 && jvParams[iCursor - 1] == jss::strict)
-        {
-            bStrict = true;
-            --iCursor;
-        }
-
-        if (!parseBase58<PublicKey>(TokenType::AccountPublic, strIdent) &&
-            !parseBase58<AccountID>(strIdent) && !parseGenericSeed(strIdent))
+        if (!parseBase58<AccountID>(strIdent))
             return rpcError(rpcACT_MALFORMED);
 
         // Get info on account.
@@ -1096,9 +1083,6 @@ private:
 
         jvRequest[jss::account] = strIdent;
 
-        if (bStrict)
-            jvRequest[jss::strict] = 1;
-
         if (iCursor == 2 && !jvParseLedger(jvRequest, jvParams[1u].asString()))
             return rpcError(rpcLGR_IDX_MALFORMED);
 
@@ -1894,7 +1878,9 @@ fromNetwork(
     constexpr auto RPC_REPLY_MAX_BYTES = megabytes(256);
 
     using namespace std::chrono_literals;
-    auto constexpr RPC_NOTIFY = 10min;
+    // auto constexpr RPC_NOTIFY = 10min; // Wietse: lolwut 10 minutes for one
+    // HTTP call?
+    auto constexpr RPC_NOTIFY = 30s;
 
     HTTPClient::request(
         bSSL,

src/ripple/net/impl/RPCSub.cpp

@@ -78,12 +78,14 @@ public:
     {
         std::lock_guard sl(mLock);
 
-        if (mDeque.size() >= eventQueueMax)
-        {
-            // Drop the previous event.
-            JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
-            mDeque.pop_back();
-        }
+        // Wietse: we're not going to limit this, this is admin-port only, scale
+        // accordingly Dropping events just like this results in inconsistent
+        // data on the receiving end if (mDeque.size() >= eventQueueMax)
+        // {
+        //     // Drop the previous event.
+        //     JLOG(j_.warn()) << "RPCCall::fromNetwork drop";
+        //     mDeque.pop_back();
+        // }
 
         auto jm = broadcast ? j_.debug() : j_.info();
         JLOG(jm) << "RPCCall::fromNetwork push: " << jvObj;
@@ -182,7 +184,8 @@ private:
     }
 
 private:
-    enum { eventQueueMax = 32 };
+    // Wietse: we're not going to limit this, this is admin-port only, scale
+    // accordingly enum { eventQueueMax = 32 };
 
     boost::asio::io_service& m_io_service;
     JobQueue& m_jobQueue;

src/ripple/nodestore/DeterministicShard.md

@@ -22,7 +22,7 @@ uint64          Appnum          Application defined constant
 uint16          KeySize         Key size in bytes
 uint64          Salt            A random seed
 uint64          Pepper          The salt hashed
-uint16          BlockSize       Size of a file block in bytes
+uint16          BlockSize       size of a file block in bytes
 uint16          LoadFactor      Target fraction in 65536ths
 uint8[56]       Reserved        Zeroes
 uint8[]         Reserved        Zero-pad to block size
@@ -160,4 +160,3 @@ Iteration 0: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71
 Iteration 1: RIPEMD160[nudb.key] = F96BF2722AB2EE009FFAE4A36AAFC4F220E21951
 Iteration 1: RIPEMD160[nudb.dat] = FAE6AE84C15968B0419FDFC014931EA12A396C71
 ```
-

src/ripple/nodestore/ShardSizeTuning.md

@@ -1,4 +1,4 @@
-# Shard Size Tuning
+# Shard size Tuning
 
 The purpose of this document is to compare the sizes of shards containing
 varying amounts of ledgers.

src/ripple/nodestore/backend/FlatmapFactory.cpp

@@ -0,0 +1,235 @@
+#include <ripple/basics/contract.h>
+#include <ripple/nodestore/Factory.h>
+#include <ripple/nodestore/Manager.h>
+#include <ripple/nodestore/impl/DecodedBlob.h>
+#include <ripple/nodestore/impl/EncodedBlob.h>
+#include <ripple/nodestore/impl/codec.h>
+#include <boost/beast/core/string.hpp>
+#include <boost/core/ignore_unused.hpp>
+#include <boost/unordered/concurrent_flat_map.hpp>
+#include <memory>
+#include <mutex>
+
+namespace ripple {
+namespace NodeStore {
+
+class FlatmapBackend : public Backend
+{
+private:
+    std::string name_;
+    beast::Journal journal_;
+    bool isOpen_{false};
+
+    struct base_uint_hasher
+    {
+        using result_type = std::size_t;
+
+        result_type
+        operator()(base_uint<256> const& value) const
+        {
+            return hardened_hash<>{}(value);
+        }
+    };
+
+    using DataStore = boost::unordered::concurrent_flat_map<
+        uint256,
+        std::vector<std::uint8_t>,  // Store compressed blob data
+        base_uint_hasher>;
+
+    DataStore table_;
+
+public:
+    FlatmapBackend(
+        size_t keyBytes,
+        Section const& keyValues,
+        beast::Journal journal)
+        : name_(get(keyValues, "path")), journal_(journal)
+    {
+        boost::ignore_unused(journal_);
+        if (name_.empty())
+            name_ = "node_db";
+    }
+
+    ~FlatmapBackend() override
+    {
+        close();
+    }
+
+    std::string
+    getName() override
+    {
+        return name_;
+    }
+
+    void
+    open(bool createIfMissing) override
+    {
+        if (isOpen_)
+            Throw<std::runtime_error>("already open");
+        isOpen_ = true;
+    }
+
+    bool
+    isOpen() override
+    {
+        return isOpen_;
+    }
+
+    void
+    close() override
+    {
+        table_.clear();
+        isOpen_ = false;
+    }
+
+    Status
+    fetch(void const* key, std::shared_ptr<NodeObject>* pObject) override
+    {
+        if (!isOpen_)
+            return notFound;
+
+        uint256 const hash(uint256::fromVoid(key));
+
+        bool found = table_.visit(hash, [&](const auto& key_value_pair) {
+            nudb::detail::buffer bf;
+            auto const result = nodeobject_decompress(
+                key_value_pair.second.data(), key_value_pair.second.size(), bf);
+            DecodedBlob decoded(hash.data(), result.first, result.second);
+            if (!decoded.wasOk())
+            {
+                *pObject = nullptr;
+                return;
+            }
+            *pObject = decoded.createObject();
+        });
+        return found ? (*pObject ? ok : dataCorrupt) : notFound;
+    }
+
+    std::pair<std::vector<std::shared_ptr<NodeObject>>, Status>
+    fetchBatch(std::vector<uint256 const*> const& hashes) override
+    {
+        std::vector<std::shared_ptr<NodeObject>> results;
+        results.reserve(hashes.size());
+        for (auto const& h : hashes)
+        {
+            std::shared_ptr<NodeObject> nObj;
+            Status status = fetch(h->begin(), &nObj);
+            if (status != ok)
+                results.push_back({});
+            else
+                results.push_back(nObj);
+        }
+        return {results, ok};
+    }
+
+    void
+    store(std::shared_ptr<NodeObject> const& object) override
+    {
+        if (!isOpen_)
+            return;
+
+        if (!object)
+            return;
+
+        EncodedBlob encoded(object);
+        nudb::detail::buffer bf;
+        auto const result =
+            nodeobject_compress(encoded.getData(), encoded.getSize(), bf);
+
+        std::vector<std::uint8_t> compressed(
+            static_cast<const std::uint8_t*>(result.first),
+            static_cast<const std::uint8_t*>(result.first) + result.second);
+
+        table_.insert_or_assign(object->getHash(), std::move(compressed));
+    }
+
+    void
+    storeBatch(Batch const& batch) override
+    {
+        for (auto const& e : batch)
+            store(e);
+    }
+
+    void
+    sync() override
+    {
+    }
+
+    void
+    for_each(std::function<void(std::shared_ptr<NodeObject>)> f) override
+    {
+        if (!isOpen_)
+            return;
+
+        table_.visit_all([&f](const auto& entry) {
+            nudb::detail::buffer bf;
+            auto const result = nodeobject_decompress(
+                entry.second.data(), entry.second.size(), bf);
+            DecodedBlob decoded(
+                entry.first.data(), result.first, result.second);
+            if (decoded.wasOk())
+                f(decoded.createObject());
+        });
+    }
+
+    int
+    getWriteLoad() override
+    {
+        return 0;
+    }
+
+    void
+    setDeletePath() override
+    {
+        close();
+    }
+
+    int
+    fdRequired() const override
+    {
+        return 0;
+    }
+
+private:
+    size_t
+    size() const
+    {
+        return table_.size();
+    }
+};
+
+class FlatmapFactory : public Factory
+{
+public:
+    FlatmapFactory()
+    {
+        Manager::instance().insert(*this);
+    }
+
+    ~FlatmapFactory() override
+    {
+        Manager::instance().erase(*this);
+    }
+
+    std::string
+    getName() const override
+    {
+        return "Flatmap";
+    }
+
+    std::unique_ptr<Backend>
+    createInstance(
+        size_t keyBytes,
+        Section const& keyValues,
+        std::size_t burstSize,
+        Scheduler& scheduler,
+        beast::Journal journal) override
+    {
+        return std::make_unique<FlatmapBackend>(keyBytes, keyValues, journal);
+    }
+};
+
+static FlatmapFactory flatmapFactory;
+
+}  // namespace NodeStore
+}  // namespace ripple

src/ripple/nodestore/backend/MemoryFactory.cpp

@@ -94,7 +94,7 @@ public:
     {
         boost::ignore_unused(journal_);  // Keep unused journal_ just in case.
         if (name_.empty())
-            Throw<std::runtime_error>("Missing path in Memory backend");
+            Throw<std::runtime_error>("Missing path in TestMemory backend");
     }
 
     ~MemoryBackend() override