Prevent consensus from getting stuck in the establish phase (#5277)

- Detects if the consensus process is "stalled". If it is, then we can declare a
  consensus and end successfully even if we do not have 80% agreement on
  our proposal.
  - "Stalled" is defined as:
    - We have a close time consensus
    - Each disputed transaction is individually stalled:
      - It has been in the final "stuck" 95% requirement for at least 2
        (avMIN_ROUNDS) "inner rounds" of phaseEstablish,
      - and either all of the other trusted proposers or this validator, if proposing,
        have had the same vote(s) for at least 4 (avSTALLED_ROUNDS) "inner
        rounds", and at least 80% of the validators (including this one, if
        appropriate) agree about the vote (whether yes or no).
- If we have been in the establish phase for more than 10x the previous
  consensus establish phase's time, then consensus is considered "expired",
  and we will leave the round, which sends a partial validation (indicating
  that the node is moving on without validating). Two restrictions avoid
  prematurely exiting, or having an extended exit in extreme situations.
  - The 10x time is clamped to be within a range of 15s
    (ledgerMAX_CONSENSUS) to 120s (ledgerABANDON_CONSENSUS).
  - If consensus has not had an opportunity to walk through all avalanche
    states (defined as not going through 8 "inner rounds" of phaseEstablish),
    then ConsensusState::Expired is treated as ConsensusState::No.
- When enough nodes leave the round, any remaining nodes will see they've
  fallen behind, and move on, too, generally before hitting the timeout. Any
  validations or partial validations sent during this time will help the
  consensus process bring the nodes back together.

.clang-format

@@ -44,6 +44,7 @@ DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
 ForEachMacros:   [ Q_FOREACH,  BOOST_FOREACH ]
+IncludeBlocks:   Regroup
 IncludeCategories:
   - Regex:           '^<(test)/'
     Priority:        0
@@ -53,8 +54,12 @@ IncludeCategories:
     Priority:        2
   - Regex:           '^<(boost)/'
     Priority:        3
-  - Regex:           '.*'
+  - Regex:           '^.*/'
     Priority:        4
+  - Regex:           '^.*\.h'
+    Priority:        5
+  - Regex:           '.*'
+    Priority:        6
 IncludeIsMainRegex: '$'
 IndentCaseLabels: true
 IndentFunctionDeclarationAfterType: false

.github/actions/xahau-ga-dependencies/action.yml

@@ -134,10 +134,17 @@ runs:
     - name: Export custom recipes
       shell: bash
       run: |
-        conan export external/snappy --version 1.1.10 --user xahaud --channel stable
-        conan export external/soci --version 4.0.3 --user xahaud --channel stable
-        conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable
-
+        # Export snappy if not already exported
+        conan list snappy/1.1.10@xahaud/stable 2>/dev/null | (grep -q "not found" && exit 1 || exit 0) || \
+          conan export external/snappy --version 1.1.10 --user xahaud --channel stable
+        
+        # Export soci if not already exported
+        conan list soci/4.0.3@xahaud/stable 2>/dev/null | (grep -q "not found" && exit 1 || exit 0) || \
+          conan export external/soci --version 4.0.3 --user xahaud --channel stable
+        
+        # Export wasmedge if not already exported
+        conan list wasmedge/0.11.2@xahaud/stable 2>/dev/null | (grep -q "not found" && exit 1 || exit 0) || \
+          conan export external/wasmedge --version 0.11.2 --user xahaud --channel stable
     - name: Install dependencies
       shell: bash
       env:

.github/workflows/xahau-ga-macos.yml

@@ -43,14 +43,22 @@ jobs:
       # To isolate environments for each Runner, instead of installing globally with brew, 
       # use mise to isolate environments for each Runner directory.
       - name: Setup toolchain (mise)
-        uses: jdx/mise-action@v2
+        uses: jdx/mise-action@v3.6.1
         with:
+          cache: false
           install: true
+          mise_toml: |
+            [tools]
+            cmake = "3.25.3"
+            python = "3.12"
+            pipx = "latest"
+            conan = "2"
+            ninja = "latest"
+            ccache = "latest"
 
       - name: Install tools via mise
         run: |
           mise install
-          mise use cmake@3.25.3 python@3.12 pipx@latest conan@2 ninja@latest ccache@latest
           mise reshim
           echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"
 

Builds/levelization/results/loops.txt

@@ -29,7 +29,7 @@ Loop: xrpld.app xrpld.overlay
   xrpld.overlay ~= xrpld.app
 
 Loop: xrpld.app xrpld.peerfinder
-  xrpld.app > xrpld.peerfinder
+  xrpld.peerfinder ~= xrpld.app
 
 Loop: xrpld.app xrpld.rpc
   xrpld.rpc > xrpld.app
@@ -44,7 +44,7 @@ Loop: xrpld.core xrpld.perflog
   xrpld.perflog == xrpld.core
 
 Loop: xrpld.net xrpld.rpc
-  xrpld.rpc > xrpld.net
+  xrpld.rpc ~= xrpld.net
 
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay

Builds/levelization/results/ordering.txt

@@ -7,6 +7,7 @@ libxrpl.protocol > xrpl.hook
 libxrpl.protocol > xrpl.json
 libxrpl.protocol > xrpl.protocol
 libxrpl.resource > xrpl.basics
+libxrpl.resource > xrpl.json
 libxrpl.resource > xrpl.resource
 libxrpl.server > xrpl.basics
 libxrpl.server > xrpl.json
@@ -43,6 +44,7 @@ test.consensus > xrpld.app
 test.consensus > xrpld.consensus
 test.consensus > xrpld.core
 test.consensus > xrpld.ledger
+test.consensus > xrpl.json
 test.consensus > xrpl.protocol
 test.core > test.jtx
 test.core > test.toplevel
@@ -60,7 +62,6 @@ test.json > test.jtx
 test.json > xrpl.json
 test.jtx > xrpl.basics
 test.jtx > xrpld.app
-test.jtx > xrpld.consensus
 test.jtx > xrpld.core
 test.jtx > xrpld.ledger
 test.jtx > xrpld.net
@@ -166,7 +167,6 @@ xrpld.core > xrpl.basics
 xrpld.core > xrpl.json
 xrpld.core > xrpl.protocol
 xrpld.ledger > xrpl.basics
-xrpld.ledger > xrpld.core
 xrpld.ledger > xrpl.json
 xrpld.ledger > xrpl.protocol
 xrpld.net > xrpl.basics
@@ -191,7 +191,6 @@ xrpld.peerfinder > xrpld.core
 xrpld.peerfinder > xrpl.protocol
 xrpld.perflog > xrpl.basics
 xrpld.perflog > xrpl.json
-xrpld.perflog > xrpl.protocol
 xrpld.rpc > xrpl.basics
 xrpld.rpc > xrpld.core
 xrpld.rpc > xrpld.ledger

conanfile.py

@@ -123,7 +123,7 @@ class Xrpl(ConanFile):
         if self.options.jemalloc:
             self.requires('jemalloc/5.3.0')
         if self.options.rocksdb:
-            self.requires('rocksdb/6.29.5')
+            self.requires('rocksdb/9.7.3')
 
     exports_sources = (
         'CMakeLists.txt',

examples/example/src/example.cpp

@@ -1,7 +1,7 @@
-#include <cstdio>
-
 #include <xrpl/protocol/BuildInfo.h>
 
+#include <cstdio>
+
 int main(int argc, char const** argv) {
     std::printf("%s\n", ripple::BuildInfo::getVersionString().c_str());
     return 0;

external/rocksdb/conandata.yml

@@ -1,27 +1,12 @@
 sources:
-  "6.29.5":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz"
-    sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0"
-  "6.27.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz"
-    sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58"
-  "6.20.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz"
-    sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca"
-  "8.8.1":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz"
-    sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e"
+  "9.7.3":
+    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v9.7.3.tar.gz"
+    sha256: "acfabb989cbfb5b5c4d23214819b059638193ec33dad2d88373c46448d16d38b"
 patches:
-  "6.29.5":
-    - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
-      patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"
-    - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch"
+  "9.7.3":
+    - patch_file: "patches/9.x.x-0001-exclude-thirdparty.patch"
       patch_description: "Do not include thirdparty.inc"
       patch_type: "portability"
-  "6.27.3":
-    - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
+    - patch_file: "patches/9.7.3-0001-memory-leak.patch"
+      patch_description: "Fix a leak of obsolete blob files left open until DB::Close()"
       patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"

external/rocksdb/conanfile.py

@@ -15,10 +15,10 @@ required_conan_version = ">=1.53.0"
 
 class RocksDBConan(ConanFile):
     name = "rocksdb"
-    homepage = "https://github.com/facebook/rocksdb"
+    description = "A library that provides an embeddable, persistent key-value store for fast storage"
     license = ("GPL-2.0-only", "Apache-2.0")
     url = "https://github.com/conan-io/conan-center-index"
-    description = "A library that provides an embeddable, persistent key-value store for fast storage"
+    homepage = "https://github.com/facebook/rocksdb"
     topics = ("database", "leveldb", "facebook", "key-value")
     package_type = "library"
     settings = "os", "arch", "compiler", "build_type"
@@ -58,12 +58,12 @@ class RocksDBConan(ConanFile):
     @property
     def _compilers_minimum_version(self):
         return {} if self._min_cppstd == "11" else {
-                "apple-clang": "10",
-                "clang": "7",
-                "gcc": "7",
-                "msvc": "191",
-                "Visual Studio": "15",
-            }
+            "apple-clang": "10",
+            "clang": "7",
+            "gcc": "7",
+            "msvc": "191",
+            "Visual Studio": "15",
+        }
 
     def export_sources(self):
         export_conandata_patches(self)
@@ -115,9 +115,9 @@ class RocksDBConan(ConanFile):
         check_min_vs(self, "191")
 
         if self.version == "6.20.3" and \
-           self.settings.os == "Linux" and \
-           self.settings.compiler == "gcc" and \
-           Version(self.settings.compiler.version) < "5":
+                self.settings.os == "Linux" and \
+                self.settings.compiler == "gcc" and \
+                Version(self.settings.compiler.version) < "5":
             raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522
 
     def source(self):
@@ -163,6 +163,8 @@ class RocksDBConan(ConanFile):
         if self.options.with_jemalloc:
             deps.set_property("jemalloc", "cmake_file_name", "JeMalloc")
             deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc")
+        if self.options.with_zstd:
+            deps.set_property("zstd", "cmake_target_name", "zstd::zstd")
         deps.generate()
 
     def build(self):

external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch

@@ -1,30 +0,0 @@
---- a/include/rocksdb/utilities/checkpoint.h
-+++ b/include/rocksdb/utilities/checkpoint.h
-@@ -8,6 +8,7 @@
- #pragma once
- #ifndef ROCKSDB_LITE
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- #include "rocksdb/status.h"
---- a/table/block_based/data_block_hash_index.h
-+++ b/table/block_based/data_block_hash_index.h
-@@ -5,6 +5,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- 
---- a/util/string_util.h
-+++ b/util/string_util.h
-@@ -6,6 +6,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <sstream>
- #include <string>
- #include <unordered_map>

external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch

@@ -1,16 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index ec59d4491..35577c998 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -101 +100,0 @@ if(MSVC)
--  option(WITH_GFLAGS "build with GFlags" OFF)
-@@ -103,2 +102,2 @@ if(MSVC)
--  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
--else()
-+endif()
-+
-@@ -117 +116 @@ else()
--  if(MINGW)
-+  if(MINGW OR MSVC)
-@@ -183 +181,0 @@ else()
--endif()

external/rocksdb/patches/9.7.3-0001-memory-leak.patch

@@ -0,0 +1,319 @@
+diff --git a/HISTORY.md b/HISTORY.md
+index 36d472229..05ad1a202 100644
+--- a/HISTORY.md
++++ b/HISTORY.md
+@@ -1,6 +1,10 @@
+ # Rocksdb Change Log
+ > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
+ 
++## 9.7.4 (10/31/2024)
++### Bug Fixes
++* Fix a leak of obsolete blob files left open until DB::Close(). This bug was introduced in version 9.4.0.
++
+ ## 9.7.3 (10/16/2024)
+ ### Behavior Changes
+ * OPTIONS file to be loaded by remote worker is now preserved so that it does not get purged by the primary host. A similar technique as how we are preserving new SST files from getting purged is used for this. min_options_file_numbers_ is tracked like pending_outputs_ is tracked.
+diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc
+index 5f340aadf..1b9faa238 100644
+--- a/db/blob/blob_file_cache.cc
++++ b/db/blob/blob_file_cache.cc
+@@ -42,6 +42,7 @@ Status BlobFileCache::GetBlobFileReader(
+   assert(blob_file_reader);
+   assert(blob_file_reader->IsEmpty());
+ 
++  // NOTE: sharing same Cache with table_cache
+   const Slice key = GetSliceForKey(&blob_file_number);
+ 
+   assert(cache_);
+@@ -98,4 +99,13 @@ Status BlobFileCache::GetBlobFileReader(
+   return Status::OK();
+ }
+ 
++void BlobFileCache::Evict(uint64_t blob_file_number) {
++  // NOTE: sharing same Cache with table_cache
++  const Slice key = GetSliceForKey(&blob_file_number);
++
++  assert(cache_);
++
++  cache_.get()->Erase(key);
++}
++
+ }  // namespace ROCKSDB_NAMESPACE
+diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h
+index 740e67ada..6858d012b 100644
+--- a/db/blob/blob_file_cache.h
++++ b/db/blob/blob_file_cache.h
+@@ -36,6 +36,15 @@ class BlobFileCache {
+                            uint64_t blob_file_number,
+                            CacheHandleGuard<BlobFileReader>* blob_file_reader);
+ 
++  // Called when a blob file is obsolete to ensure it is removed from the cache
++  // to avoid effectively leaking the open file and assicated memory
++  void Evict(uint64_t blob_file_number);
++
++  // Used to identify cache entries for blob files (not normally useful)
++  static const Cache::CacheItemHelper* GetHelper() {
++    return CacheInterface::GetBasicHelper();
++  }
++
+  private:
+   using CacheInterface =
+       BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
+diff --git a/db/column_family.h b/db/column_family.h
+index e4b7adde8..86637736a 100644
+--- a/db/column_family.h
++++ b/db/column_family.h
+@@ -401,6 +401,7 @@ class ColumnFamilyData {
+                          SequenceNumber earliest_seq);
+ 
+   TableCache* table_cache() const { return table_cache_.get(); }
++  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
+   BlobSource* blob_source() const { return blob_source_.get(); }
+ 
+   // See documentation in compaction_picker.h
+diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc
+index 261593423..06573ac2e 100644
+--- a/db/db_impl/db_impl.cc
++++ b/db/db_impl/db_impl.cc
+@@ -659,8 +659,9 @@ Status DBImpl::CloseHelper() {
+   // We need to release them before the block cache is destroyed. The block
+   // cache may be destroyed inside versions_.reset(), when column family data
+   // list is destroyed, so leaving handles in table cache after
+-  // versions_.reset() may cause issues.
+-  // Here we clean all unreferenced handles in table cache.
++  // versions_.reset() may cause issues. Here we clean all unreferenced handles
++  // in table cache, and (for certain builds/conditions) assert that no obsolete
++  // files are hanging around unreferenced (leak) in the table/blob file cache.
+   // Now we assume all user queries have finished, so only version set itself
+   // can possibly hold the blocks from block cache. After releasing unreferenced
+   // handles here, only handles held by version set left and inside
+@@ -668,6 +669,9 @@ Status DBImpl::CloseHelper() {
+   // time a handle is released, we erase it from the cache too. By doing that,
+   // we can guarantee that after versions_.reset(), table cache is empty
+   // so the cache can be safely destroyed.
++#ifndef NDEBUG
++  TEST_VerifyNoObsoleteFilesCached(/*db_mutex_already_held=*/true);
++#endif  // !NDEBUG
+   table_cache_->EraseUnRefEntries();
+ 
+   for (auto& txn_entry : recovered_transactions_) {
+@@ -3227,6 +3231,8 @@ Status DBImpl::MultiGetImpl(
+       s = Status::Aborted();
+       break;
+     }
++    // This could be a long-running operation
++    ROCKSDB_THREAD_YIELD_HOOK();
+   }
+ 
+   // Post processing (decrement reference counts and record statistics)
+diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h
+index 5e4fa310b..ccc0abfa7 100644
+--- a/db/db_impl/db_impl.h
++++ b/db/db_impl/db_impl.h
+@@ -1241,9 +1241,14 @@ class DBImpl : public DB {
+   static Status TEST_ValidateOptions(const DBOptions& db_options) {
+     return ValidateOptions(db_options);
+   }
+-
+ #endif  // NDEBUG
+ 
++  // In certain configurations, verify that the table/blob file cache only
++  // contains entries for live files, to check for effective leaks of open
++  // files. This can only be called when purging of obsolete files has
++  // "settled," such as during parts of DB Close().
++  void TEST_VerifyNoObsoleteFilesCached(bool db_mutex_already_held) const;
++
+   // persist stats to column family "_persistent_stats"
+   void PersistStats();
+ 
+diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc
+index 790a50d7a..67f5b4aaf 100644
+--- a/db/db_impl/db_impl_debug.cc
++++ b/db/db_impl/db_impl_debug.cc
+@@ -9,6 +9,7 @@
+ 
+ #ifndef NDEBUG
+ 
++#include "db/blob/blob_file_cache.h"
+ #include "db/column_family.h"
+ #include "db/db_impl/db_impl.h"
+ #include "db/error_handler.h"
+@@ -328,5 +329,49 @@ size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
+   InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
+   return EstimateInMemoryStatsHistorySize();
+ }
++
++void DBImpl::TEST_VerifyNoObsoleteFilesCached(
++    bool db_mutex_already_held) const {
++  // This check is somewhat expensive and obscure to make a part of every
++  // unit test in every build variety. Thus, we only enable it for ASAN builds.
++  if (!kMustFreeHeapAllocations) {
++    return;
++  }
++
++  std::optional<InstrumentedMutexLock> l;
++  if (db_mutex_already_held) {
++    mutex_.AssertHeld();
++  } else {
++    l.emplace(&mutex_);
++  }
++
++  std::vector<uint64_t> live_files;
++  for (auto cfd : *versions_->GetColumnFamilySet()) {
++    if (cfd->IsDropped()) {
++      continue;
++    }
++    // Sneakily add both SST and blob files to the same list
++    cfd->current()->AddLiveFiles(&live_files, &live_files);
++  }
++  std::sort(live_files.begin(), live_files.end());
++
++  auto fn = [&live_files](const Slice& key, Cache::ObjectPtr, size_t,
++                          const Cache::CacheItemHelper* helper) {
++    if (helper != BlobFileCache::GetHelper()) {
++      // Skip non-blob files for now
++      // FIXME: diagnose and fix the leaks of obsolete SST files revealed in
++      // unit tests.
++      return;
++    }
++    // See TableCache and BlobFileCache
++    assert(key.size() == sizeof(uint64_t));
++    uint64_t file_number;
++    GetUnaligned(reinterpret_cast<const uint64_t*>(key.data()), &file_number);
++    // Assert file is in sorted live_files
++    assert(
++        std::binary_search(live_files.begin(), live_files.end(), file_number));
++  };
++  table_cache_->ApplyToAllEntries(fn, {});
++}
+ }  // namespace ROCKSDB_NAMESPACE
+ #endif  // NDEBUG
+diff --git a/db/db_iter.cc b/db/db_iter.cc
+index e02586377..bf4749eb9 100644
+--- a/db/db_iter.cc
++++ b/db/db_iter.cc
+@@ -540,6 +540,8 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
+     } else {
+       iter_.Next();
+     }
++    // This could be a long-running operation due to tombstones, etc.
++    ROCKSDB_THREAD_YIELD_HOOK();
+   } while (iter_.Valid());
+ 
+   valid_ = false;
+diff --git a/db/table_cache.cc b/db/table_cache.cc
+index 71fc29c32..8a5be75e8 100644
+--- a/db/table_cache.cc
++++ b/db/table_cache.cc
+@@ -164,6 +164,7 @@ Status TableCache::GetTableReader(
+ }
+ 
+ Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&file_number);
+   return cache->Lookup(key);
+ }
+@@ -179,6 +180,7 @@ Status TableCache::FindTable(
+     size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) {
+   PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock);
+   uint64_t number = file_meta.fd.GetNumber();
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&number);
+   *handle = cache_.Lookup(key);
+   TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
+diff --git a/db/version_builder.cc b/db/version_builder.cc
+index ed8ab8214..c98f53f42 100644
+--- a/db/version_builder.cc
++++ b/db/version_builder.cc
+@@ -24,6 +24,7 @@
+ #include <vector>
+ 
+ #include "cache/cache_reservation_manager.h"
++#include "db/blob/blob_file_cache.h"
+ #include "db/blob/blob_file_meta.h"
+ #include "db/dbformat.h"
+ #include "db/internal_stats.h"
+@@ -744,12 +745,9 @@ class VersionBuilder::Rep {
+       return Status::Corruption("VersionBuilder", oss.str());
+     }
+ 
+-    // Note: we use C++11 for now but in C++14, this could be done in a more
+-    // elegant way using generalized lambda capture.
+-    VersionSet* const vs = version_set_;
+-    const ImmutableCFOptions* const ioptions = ioptions_;
+-
+-    auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) {
++    auto deleter = [vs = version_set_, ioptions = ioptions_,
++                    bc = cfd_ ? cfd_->blob_file_cache()
++                              : nullptr](SharedBlobFileMetaData* shared_meta) {
+       if (vs) {
+         assert(ioptions);
+         assert(!ioptions->cf_paths.empty());
+@@ -758,6 +756,9 @@ class VersionBuilder::Rep {
+         vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(),
+                                 ioptions->cf_paths.front().path);
+       }
++      if (bc) {
++        bc->Evict(shared_meta->GetBlobFileNumber());
++      }
+ 
+       delete shared_meta;
+     };
+@@ -766,7 +767,7 @@ class VersionBuilder::Rep {
+         blob_file_number, blob_file_addition.GetTotalBlobCount(),
+         blob_file_addition.GetTotalBlobBytes(),
+         blob_file_addition.GetChecksumMethod(),
+-        blob_file_addition.GetChecksumValue(), deleter);
++        blob_file_addition.GetChecksumValue(), std::move(deleter));
+ 
+     mutable_blob_file_metas_.emplace(
+         blob_file_number, MutableBlobFileMetaData(std::move(shared_meta)));
+diff --git a/db/version_set.h b/db/version_set.h
+index 9336782b1..024f869e7 100644
+--- a/db/version_set.h
++++ b/db/version_set.h
+@@ -1514,7 +1514,6 @@ class VersionSet {
+   void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+ 
+   void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
+-    // TODO: Erase file from BlobFileCache?
+     obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
+   }
+ 
+diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h
+index 2a19796b8..0afa2cab1 100644
+--- a/include/rocksdb/version.h
++++ b/include/rocksdb/version.h
+@@ -13,7 +13,7 @@
+ // minor or major version number planned for release.
+ #define ROCKSDB_MAJOR 9
+ #define ROCKSDB_MINOR 7
+-#define ROCKSDB_PATCH 3
++#define ROCKSDB_PATCH 4
+ 
+ // Do not use these. We made the mistake of declaring macros starting with
+ // double underscore. Now we have to live with our choice. We'll deprecate these
+diff --git a/port/port.h b/port/port.h
+index 13aa56d47..141716e5b 100644
+--- a/port/port.h
++++ b/port/port.h
+@@ -19,3 +19,19 @@
+ #elif defined(OS_WIN)
+ #include "port/win/port_win.h"
+ #endif
++
++#ifdef OS_LINUX
++// A temporary hook into long-running RocksDB threads to support modifying their
++// priority etc. This should become a public API hook once the requirements
++// are better understood.
++extern "C" void RocksDbThreadYield() __attribute__((__weak__));
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {                                 \
++    if (RocksDbThreadYield) {       \
++      RocksDbThreadYield();         \
++    }                               \
++  }
++#else
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {}
++#endif

external/rocksdb/patches/9.x.x-0001-exclude-thirdparty.patch

@@ -0,0 +1,30 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 93b884d..b715cb6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -106,14 +106,9 @@ endif()
+ include(CMakeDependentOption)
+
+ if(MSVC)
+-  option(WITH_GFLAGS "build with GFlags" OFF)
+   option(WITH_XPRESS "build with windows built in compression" OFF)
+-  option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF)
+-
+-  if(NOT ROCKSDB_SKIP_THIRDPARTY)
+-    include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+-  endif()
+-else()
++endif()
++if(TRUE)
+   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
+     # FreeBSD has jemalloc as default malloc
+     # but it does not have all the jemalloc files in include/...
+@@ -126,7 +121,7 @@ else()
+     endif()
+   endif()
+
+-  if(MINGW)
++  if(MSVC OR MINGW)
+     option(WITH_GFLAGS "build with GFlags" OFF)
+   else()
+     option(WITH_GFLAGS "build with GFlags" ON)

hook/hookapi.h

@@ -45,8 +45,8 @@
 
 #include "error.h"
 #include "extern.h"
-#include "sfcodes.h"
 #include "macro.h"
+#include "sfcodes.h"
 #include "tts.h"
 
 #endif

include/xrpl/basics/BasicConfig.h

@@ -27,7 +27,6 @@
 
 #include <algorithm>
 #include <optional>
-#include <ostream>
 #include <string>
 #include <unordered_map>
 #include <vector>

include/xrpl/basics/Buffer.h

@@ -22,10 +22,9 @@
 
 #include <xrpl/basics/Slice.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <cstdint>
 #include <cstring>
-#include <memory>
-#include <utility>
 
 namespace ripple {
 

include/xrpl/basics/CompressionAlgorithms.h

@@ -21,9 +21,11 @@
 #define RIPPLED_COMPRESSIONALGORITHMS_H_INCLUDED
 
 #include <xrpl/basics/contract.h>
+
+#include <lz4.h>
+
 #include <algorithm>
 #include <cstdint>
-#include <lz4.h>
 #include <stdexcept>
 #include <vector>
 

include/xrpl/basics/CountedObject.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_COUNTEDOBJECT_H_INCLUDED
 
 #include <xrpl/beast/type_name.h>
+
 #include <atomic>
 #include <string>
 #include <utility>

include/xrpl/basics/Expected.h

@@ -24,9 +24,7 @@
 
 #include <boost/outcome.hpp>
 
-#include <concepts>
 #include <stdexcept>
-#include <type_traits>
 
 namespace ripple {
 

include/xrpl/basics/LocalValue.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_LOCALVALUE_H_INCLUDED
 
 #include <boost/thread/tss.hpp>
+
 #include <memory>
 #include <unordered_map>
 

include/xrpl/basics/Log.h

@@ -22,8 +22,10 @@
 
 #include <xrpl/basics/UnorderedContainers.h>
 #include <xrpl/beast/utility/Journal.h>
+
 #include <boost/beast/core/string.hpp>
 #include <boost/filesystem.hpp>
+
 #include <fstream>
 #include <map>
 #include <memory>

include/xrpl/basics/Resolver.h

@@ -20,11 +20,11 @@
 #ifndef RIPPLE_BASICS_RESOLVER_H_INCLUDED
 #define RIPPLE_BASICS_RESOLVER_H_INCLUDED
 
+#include <xrpl/beast/net/IPEndpoint.h>
+
 #include <functional>
 #include <vector>
 
-#include <xrpl/beast/net/IPEndpoint.h>
-
 namespace ripple {
 
 class Resolver

include/xrpl/basics/ResolverAsio.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/Resolver.h>
 #include <xrpl/beast/utility/Journal.h>
+
 #include <boost/asio/io_service.hpp>
 
 namespace ripple {

include/xrpl/basics/Slice.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <algorithm>
 #include <array>
 #include <cstdint>

include/xrpl/basics/TaggedCache.h

@@ -25,6 +25,7 @@
 #include <xrpl/basics/hardened_hash.h>
 #include <xrpl/beast/clock/abstract_clock.h>
 #include <xrpl/beast/insight/Insight.h>
+
 #include <atomic>
 #include <functional>
 #include <mutex>

include/xrpl/basics/UnorderedContainers.h

@@ -25,6 +25,7 @@
 #include <xrpl/beast/hash/hash_append.h>
 #include <xrpl/beast/hash/uhash.h>
 #include <xrpl/beast/hash/xxhasher.h>
+
 #include <unordered_map>
 #include <unordered_set>
 

include/xrpl/basics/base_uint.h

@@ -33,12 +33,13 @@
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/Zero.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/endian/conversion.hpp>
 #include <boost/functional/hash.hpp>
+
 #include <algorithm>
 #include <array>
 #include <cstring>
-#include <functional>
 #include <type_traits>
 
 namespace ripple {

include/xrpl/basics/chrono.h

@@ -20,17 +20,16 @@
 #ifndef RIPPLE_BASICS_CHRONO_H_INCLUDED
 #define RIPPLE_BASICS_CHRONO_H_INCLUDED
 
-#include <date/date.h>
-
 #include <xrpl/beast/clock/abstract_clock.h>
 #include <xrpl/beast/clock/basic_seconds_clock.h>
 #include <xrpl/beast/clock/manual_clock.h>
 
+#include <date/date.h>
+
 #include <chrono>
 #include <cstdint>
 #include <ratio>
 #include <string>
-#include <type_traits>
 
 namespace ripple {
 

include/xrpl/basics/contract.h

@@ -21,9 +21,9 @@
 #define RIPPLE_BASICS_CONTRACT_H_INCLUDED
 
 #include <xrpl/beast/type_name.h>
+
 #include <exception>
 #include <string>
-#include <typeinfo>
 #include <utility>
 
 namespace ripple {

include/xrpl/basics/make_SSLContext.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_MAKE_SSLCONTEXT_H_INCLUDED
 
 #include <boost/asio/ssl/context.hpp>
+
 #include <string>
 
 namespace ripple {

include/xrpl/basics/partitioned_unordered_map.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/beast/hash/uhash.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <functional>
 #include <optional>
 #include <string>

include/xrpl/basics/random.h

@@ -22,9 +22,9 @@
 
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/beast/xor_shift_engine.h>
+
 #include <cstddef>
 #include <cstdint>
-#include <cstring>
 #include <limits>
 #include <mutex>
 #include <random>

include/xrpl/basics/spinlock.h

@@ -19,6 +19,7 @@
 #define RIPPLE_BASICS_SPINLOCK_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <atomic>
 #include <limits>
 #include <type_traits>

include/xrpl/basics/tagged_integer.h

@@ -21,7 +21,9 @@
 #define BEAST_UTILITY_TAGGED_INTEGER_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <boost/operators.hpp>
+
 #include <functional>
 #include <iostream>
 #include <type_traits>

include/xrpl/beast/asio/io_latency_probe.h

@@ -21,6 +21,7 @@
 #define BEAST_ASIO_IO_LATENCY_PROBE_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/basic_waitable_timer.hpp>
 #include <boost/asio/io_service.hpp>
 

include/xrpl/beast/container/detail/aged_ordered_container.h

@@ -25,9 +25,11 @@
 #include <xrpl/beast/container/detail/aged_associative_container.h>
 #include <xrpl/beast/container/detail/aged_container_iterator.h>
 #include <xrpl/beast/container/detail/empty_base_optimization.h>
+
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/set.hpp>
 #include <boost/version.hpp>
+
 #include <algorithm>
 #include <functional>
 #include <initializer_list>

include/xrpl/beast/container/detail/aged_unordered_container.h

@@ -25,8 +25,10 @@
 #include <xrpl/beast/container/detail/aged_associative_container.h>
 #include <xrpl/beast/container/detail/aged_container_iterator.h>
 #include <xrpl/beast/container/detail/empty_base_optimization.h>
+
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/unordered_set.hpp>
+
 #include <algorithm>
 #include <cmath>
 #include <functional>

include/xrpl/beast/container/detail/empty_base_optimization.h

@@ -11,6 +11,7 @@
 #define BEAST_CONTAINER_DETAIL_EMPTY_BASE_OPTIMIZATION_H_INCLUDED
 
 #include <boost/type_traits/is_final.hpp>
+
 #include <type_traits>
 #include <utility>
 

include/xrpl/beast/core/LexicalCast.h

@@ -23,6 +23,7 @@
 #include <xrpl/beast/utility/instrumentation.h>
 
 #include <boost/core/detail/string_view.hpp>
+
 #include <algorithm>
 #include <cerrno>
 #include <charconv>

include/xrpl/beast/core/List.h

@@ -21,7 +21,6 @@
 #define BEAST_INTRUSIVE_LIST_H_INCLUDED
 
 #include <iterator>
-#include <type_traits>
 
 namespace beast {
 

include/xrpl/beast/hash/hash_append.h

@@ -23,6 +23,7 @@
 
 #include <boost/container/flat_set.hpp>
 #include <boost/endian/conversion.hpp>
+
 #include <array>
 #include <chrono>
 #include <cstdint>

include/xrpl/beast/hash/xxhasher.h

@@ -21,6 +21,7 @@
 #define BEAST_HASH_XXHASHER_H_INCLUDED
 
 #include <boost/endian/conversion.hpp>
+
 #include <xxhash.h>
 
 #include <cstddef>

include/xrpl/beast/insight/Meter.h

@@ -20,10 +20,10 @@
 #ifndef BEAST_INSIGHT_METER_H_INCLUDED
 #define BEAST_INSIGHT_METER_H_INCLUDED
 
-#include <memory>
-
 #include <xrpl/beast/insight/MeterImpl.h>
 
+#include <memory>
+
 namespace beast {
 namespace insight {
 

include/xrpl/beast/insight/StatsDCollector.h

@@ -21,7 +21,6 @@
 #define BEAST_INSIGHT_STATSDCOLLECTOR_H_INCLUDED
 
 #include <xrpl/beast/insight/Collector.h>
-
 #include <xrpl/beast/net/IPEndpoint.h>
 #include <xrpl/beast/utility/Journal.h>
 

include/xrpl/beast/net/IPAddress.h

@@ -25,8 +25,10 @@
 #include <xrpl/beast/net/IPAddressV4.h>
 #include <xrpl/beast/net/IPAddressV6.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/ip/address.hpp>
 #include <boost/functional/hash.hpp>
+
 #include <cstdint>
 #include <ios>
 #include <sstream>

include/xrpl/beast/net/IPAddressConversion.h

@@ -22,8 +22,6 @@
 
 #include <xrpl/beast/net/IPEndpoint.h>
 
-#include <sstream>
-
 #include <boost/asio.hpp>
 
 namespace beast {

include/xrpl/beast/net/IPAddressV4.h

@@ -21,7 +21,9 @@
 #define BEAST_NET_IPADDRESSV4_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <boost/asio/ip/address_v4.hpp>
+
 #include <cstdint>
 #include <functional>
 #include <ios>

include/xrpl/beast/net/IPAddressV6.h

@@ -21,7 +21,9 @@
 #define BEAST_NET_IPADDRESSV6_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/ip/address_v6.hpp>
+
 #include <cstdint>
 #include <functional>
 #include <ios>

include/xrpl/beast/rfc2616.h

@@ -20,10 +20,6 @@
 #ifndef BEAST_RFC2616_HPP
 #define BEAST_RFC2616_HPP
 
-// TODO: This include <cstdint> is a workaround for beast compilation bug.
-// Remove when fix https://github.com/boostorg/beast/pull/2682/ is available.
-#include <cstdint>
-
 #include <boost/beast/http/message.hpp>
 #include <boost/beast/http/rfc7230.hpp>
 #include <boost/range/algorithm/equal.hpp>
@@ -32,6 +28,7 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cstdint>
 #include <iterator>
 #include <string>
 #include <utility>

include/xrpl/beast/test/yield_to.h

@@ -11,6 +11,7 @@
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/optional.hpp>
+
 #include <condition_variable>
 #include <functional>
 #include <mutex>

include/xrpl/beast/unit_test/match.h

@@ -9,6 +9,7 @@
 #define BEAST_UNIT_TEST_MATCH_HPP
 
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <string>
 
 namespace beast {

include/xrpl/beast/unit_test/reporter.h

@@ -10,8 +10,10 @@
 
 #include <xrpl/beast/unit_test/amount.h>
 #include <xrpl/beast/unit_test/recorder.h>
+
 #include <boost/lexical_cast.hpp>
 #include <boost/optional.hpp>
+
 #include <algorithm>
 #include <chrono>
 #include <functional>

include/xrpl/beast/unit_test/runner.h

@@ -9,7 +9,9 @@
 #define BEAST_UNIT_TEST_RUNNER_H_INCLUDED
 
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <boost/assert.hpp>
+
 #include <mutex>
 #include <ostream>
 #include <string>

include/xrpl/beast/unit_test/suite.h

@@ -9,9 +9,11 @@
 #define BEAST_UNIT_TEST_SUITE_HPP
 
 #include <xrpl/beast/unit_test/runner.h>
+
 #include <boost/filesystem.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/throw_exception.hpp>
+
 #include <ostream>
 #include <sstream>
 #include <string>

include/xrpl/beast/unit_test/suite_list.h

@@ -10,7 +10,9 @@
 
 #include <xrpl/beast/unit_test/detail/const_container.h>
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <boost/assert.hpp>
+
 #include <set>
 #include <typeindex>
 #include <unordered_set>

include/xrpl/beast/unit_test/thread.h

@@ -9,6 +9,7 @@
 #define BEAST_UNIT_TEST_THREAD_HPP
 
 #include <xrpl/beast/unit_test/suite.h>
+
 #include <functional>
 #include <thread>
 #include <utility>

include/xrpl/beast/utility/Journal.h

@@ -21,6 +21,7 @@
 #define BEAST_UTILITY_JOURNAL_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <sstream>
 
 namespace beast {

include/xrpl/beast/utility/PropertyStream.h

@@ -22,11 +22,9 @@
 
 #include <xrpl/beast/core/List.h>
 
-#include <cstdint>
 #include <mutex>
 #include <sstream>
 #include <string>
-#include <utility>
 
 namespace beast {
 

include/xrpl/beast/utility/rngfill.h

@@ -21,6 +21,7 @@
 #define BEAST_RANDOM_RNGFILL_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <array>
 #include <cstdint>
 #include <cstring>

include/xrpl/beast/utility/temp_dir.h

@@ -21,6 +21,7 @@
 #define BEAST_UTILITY_TEMP_DIR_H_INCLUDED
 
 #include <boost/filesystem.hpp>
+
 #include <string>
 
 namespace beast {

include/xrpl/crypto/csprng.h

@@ -21,8 +21,6 @@
 #define RIPPLE_CRYPTO_RANDOM_H_INCLUDED
 
 #include <mutex>
-#include <string>
-#include <type_traits>
 
 namespace ripple {
 

include/xrpl/hook/Guard.h

@@ -1,4 +1,5 @@
 #include "Enum.h"
+
 #include <functional>
 #include <iostream>
 #include <map>

include/xrpl/hook/guard_checker.cpp

@@ -1,15 +1,17 @@
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+
 #include "Guard.h"
 #include <fcntl.h>
-#include <iostream>
-#include <optional>
-#include <ostream>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
-#include <string_view>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
 #include <unistd.h>
+
+#include <iostream>
+#include <optional>
+#include <ostream>
+#include <string_view>
 #include <vector>
 
 int

include/xrpl/json/Object.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_OBJECT_H_INCLUDED
 
 #include <xrpl/json/Writer.h>
+
 #include <memory>
 
 namespace Json {

include/xrpl/json/Output.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_OUTPUT_H_INCLUDED
 
 #include <boost/beast/core/string.hpp>
+
 #include <functional>
 #include <string>
 

include/xrpl/json/Writer.h

@@ -24,6 +24,7 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/json/Output.h>
 #include <xrpl/json/json_value.h>
+
 #include <memory>
 
 namespace Json {

include/xrpl/json/detail/json_assert.h

@@ -20,6 +20,7 @@
 #ifndef RIPPLE_JSON_JSON_ASSERT_H_INCLUDED
 #define RIPPLE_JSON_JSON_ASSERT_H_INCLUDED
 
+#include <xrpl/basics/contract.h>
 #include <xrpl/json/json_errors.h>
 
 #define JSON_ASSERT_MESSAGE(condition, message) \

include/xrpl/json/json_reader.h

@@ -24,7 +24,9 @@
 
 #include <xrpl/json/json_forwards.h>
 #include <xrpl/json/json_value.h>
+
 #include <boost/asio/buffer.hpp>
+
 #include <stack>
 
 namespace Json {

include/xrpl/json/json_value.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_JSON_VALUE_H_INCLUDED
 
 #include <xrpl/json/json_forwards.h>
+
 #include <cstring>
 #include <map>
 #include <string>

include/xrpl/json/json_writer.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/json/json_forwards.h>
 #include <xrpl/json/json_value.h>
+
 #include <ostream>
 #include <vector>
 

include/xrpl/protocol/Book.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/CountedObject.h>
 #include <xrpl/protocol/Issue.h>
+
 #include <boost/utility/base_from_member.hpp>
 
 namespace ripple {

include/xrpl/protocol/Feature.h

@@ -21,7 +21,9 @@
 #define RIPPLE_PROTOCOL_FEATURE_H_INCLUDED
 
 #include <xrpl/basics/base_uint.h>
+
 #include <boost/container/flat_map.hpp>
+
 #include <array>
 #include <bitset>
 #include <map>
@@ -33,35 +35,39 @@
  *
  * Steps required to add new features to the code:
  *
- * 1) In this file, increment `numFeatures` and add a uint256 declaration
- *    for the feature at the bottom
- * 2) Add a uint256 definition for the feature to the corresponding source
- *    file (Feature.cpp). Use `registerFeature` to create the feature with
- *    the feature's name, `Supported::no`, and `VoteBehavior::DefaultNo`. This
- *    should be the only place the feature's name appears in code as a string.
- * 3) Use the uint256 as the parameter to `view.rules.enabled()` to
- *    control flow into new code that this feature limits.
- * 4) If the feature development is COMPLETE, and the feature is ready to be
- *    SUPPORTED, change the `registerFeature` parameter to Supported::yes.
- * 5) When the feature is ready to be ENABLED, change the `registerFeature`
- *    parameter to `VoteBehavior::DefaultYes`.
- * In general, any newly supported amendments (`Supported::yes`) should have
- * a `VoteBehavior::DefaultNo` for at least one full release cycle. High
- * priority bug fixes can be an exception to this rule of thumb.
+ * 1) Add the appropriate XRPL_FEATURE or XRPL_FIX macro definition for the
+ *    feature to features.macro with the feature's name, `Supported::no`, and
+ *    `VoteBehavior::DefaultNo`.
+ *
+ * 2) Use the generated variable name as the parameter to `view.rules.enabled()`
+ *    to control flow into new code that this feature limits. (featureName or
+ *    fixName)
+ *
+ * 3) If the feature development is COMPLETE, and the feature is ready to be
+ *    SUPPORTED, change the macro parameter in features.macro to Supported::yes.
+ *
+ * 4) In general, any newly supported amendments (`Supported::yes`) should have
+ *    a `VoteBehavior::DefaultNo` indefinitely so that external governance can
+ *    make the decision on when to activate it. High priority bug fixes can be
+ *    an exception to this rule. In such cases, ensure the fix has been
+ *    clearly communicated to the community using appropriate channels,
+ *    then change the macro parameter in features.macro to
+ *    `VoteBehavior::DefaultYes`. The communication process is beyond
+ *    the scope of these instructions.
+ *
  *
  * When a feature has been enabled for several years, the conditional code
  * may be removed, and the feature "retired". To retire a feature:
- * 1) Remove the uint256 declaration from this file.
- * 2) MOVE the uint256 definition in Feature.cpp to the "retired features"
- *    section at the end of the file.
- * 3) CHANGE the name of the variable to start with "retired".
- * 4) CHANGE the parameters of the `registerFeature` call to `Supported::yes`
- *    and `VoteBehavior::DefaultNo`.
+ *
+ * 1) MOVE the macro definition in features.macro to the "retired features"
+ *    section at the end of the file, and change the macro to XRPL_RETIRE.
+ *
  * The feature must remain registered and supported indefinitely because it
- * still exists in the ledger, but there is no need to vote for it because
- * there's nothing to vote for. If it is removed completely from the code, any
- * instances running that code will get amendment blocked. Removing the
- * feature from the ledger is beyond the scope of these instructions.
+ * may exist in the Amendments object on ledger. There is no need to vote
+ * for it because there's nothing to vote for. If the feature definition is
+ * removed completely from the code, any instances running that code will get
+ * amendment blocked. Removing the feature from the ledger is beyond the scope
+ * of these instructions.
  *
  */
 
@@ -76,11 +82,32 @@ allAmendments();
 
 namespace detail {
 
+#pragma push_macro("XRPL_FEATURE")
+#undef XRPL_FEATURE
+#pragma push_macro("XRPL_FIX")
+#undef XRPL_FIX
+#pragma push_macro("XRPL_RETIRE")
+#undef XRPL_RETIRE
+
+#define XRPL_FEATURE(name, supported, vote) +1
+#define XRPL_FIX(name, supported, vote) +1
+#define XRPL_RETIRE(name) +1
+
 // This value SHOULD be equal to the number of amendments registered in
 // Feature.cpp. Because it's only used to reserve storage, and determine how
 // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than
 // the actual number of amendments. A LogicError on startup will verify this.
-static constexpr std::size_t numFeatures = 110;
+static constexpr std::size_t numFeatures =
+    (0 +
+#include <xrpl/protocol/detail/features.macro>
+    );
+
+#undef XRPL_RETIRE
+#pragma pop_macro("XRPL_RETIRE")
+#undef XRPL_FIX
+#pragma pop_macro("XRPL_FIX")
+#undef XRPL_FEATURE
+#pragma pop_macro("XRPL_FEATURE")
 
 /** Amendments that this server supports and the default voting behavior.
    Whether they are enabled depends on the Rules defined in the validated
@@ -320,12 +347,17 @@ foreachFeature(FeatureBitset bs, F&& f)
 #undef XRPL_FEATURE
 #pragma push_macro("XRPL_FIX")
 #undef XRPL_FIX
+#pragma push_macro("XRPL_RETIRE")
+#undef XRPL_RETIRE
 
 #define XRPL_FEATURE(name, supported, vote) extern uint256 const feature##name;
 #define XRPL_FIX(name, supported, vote) extern uint256 const fix##name;
+#define XRPL_RETIRE(name)
 
 #include <xrpl/protocol/detail/features.macro>
 
+#undef XRPL_RETIRE
+#pragma pop_macro("XRPL_RETIRE")
 #undef XRPL_FIX
 #pragma pop_macro("XRPL_FIX")
 #undef XRPL_FEATURE

include/xrpl/protocol/FeeUnits.h

@@ -23,6 +23,7 @@
 #include <xrpl/beast/utility/Zero.h>
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/json/json_value.h>
+
 #include <boost/multiprecision/cpp_int.hpp>
 #include <boost/operators.hpp>
 

include/xrpl/protocol/HashPrefix.h

@@ -21,6 +21,7 @@
 #define RIPPLE_PROTOCOL_HASHPREFIX_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <cstdint>
 
 namespace ripple {

include/xrpl/protocol/IOUAmount.h

@@ -23,7 +23,9 @@
 #include <xrpl/basics/LocalValue.h>
 #include <xrpl/basics/Number.h>
 #include <xrpl/beast/utility/Zero.h>
+
 #include <boost/operators.hpp>
+
 #include <cstdint>
 #include <string>
 #include <utility>

include/xrpl/protocol/Import.h

@@ -22,9 +22,11 @@
 
 // #include <ripple/basics/Log.h>
 #include <xrpld/app/misc/Manifest.h>
+
 #include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base64.h>
 #include <xrpl/json/json_reader.h>
+
 #include <charconv>
 
 namespace ripple {

include/xrpl/protocol/KnownFormats.h

@@ -23,7 +23,9 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/beast/type_name.h>
 #include <xrpl/protocol/SOTemplate.h>
+
 #include <boost/container/flat_map.hpp>
+
 #include <algorithm>
 #include <forward_list>
 

include/xrpl/protocol/MultiApiJson.h

@@ -20,10 +20,10 @@
 #ifndef RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED
 #define RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED
 
+#include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/json/json_value.h>
 #include <xrpl/protocol/ApiVersion.h>
 
-#include <xrpl/beast/utility/instrumentation.h>
 #include <array>
 #include <concepts>
 #include <cstdlib>

include/xrpl/protocol/Protocol.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/ByteUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/partitioned_unordered_map.h>
+
 #include <cstdint>
 
 namespace ripple {

include/xrpl/protocol/Rate.h

@@ -22,7 +22,9 @@
 
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/protocol/STAmount.h>
+
 #include <boost/operators.hpp>
+
 #include <cstdint>
 #include <ostream>
 

include/xrpl/protocol/SOTemplate.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/contract.h>
 #include <xrpl/protocol/SField.h>
+
 #include <functional>
 #include <initializer_list>
 #include <memory>

include/xrpl/protocol/STBase.h

@@ -23,12 +23,14 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/Serializer.h>
+
 #include <memory>
 #include <ostream>
 #include <string>
 #include <type_traits>
 #include <typeinfo>
 #include <utility>
+
 namespace ripple {
 
 /// Note, should be treated as flags that can be | and &

include/xrpl/protocol/STExchange.h

@@ -28,6 +28,7 @@
 #include <xrpl/protocol/STBlob.h>
 #include <xrpl/protocol/STInteger.h>
 #include <xrpl/protocol/STObject.h>
+
 #include <memory>
 #include <optional>
 #include <stdexcept>

include/xrpl/protocol/STLedgerEntry.h

@@ -124,4 +124,4 @@ STLedgerEntry::getType() const
 
 }  // namespace ripple
 
-#endif
+#endif

include/xrpl/protocol/STObject.h

@@ -35,7 +35,9 @@
 #include <xrpl/protocol/STPathSet.h>
 #include <xrpl/protocol/STVector256.h>
 #include <xrpl/protocol/detail/STVar.h>
+
 #include <boost/iterator/transform_iterator.hpp>
+
 #include <optional>
 #include <stdexcept>
 #include <type_traits>

include/xrpl/protocol/STParsedJSON.h

@@ -21,6 +21,7 @@
 #define RIPPLE_PROTOCOL_STPARSEDJSON_H_INCLUDED
 
 #include <xrpl/protocol/STArray.h>
+
 #include <optional>
 
 namespace ripple {

include/xrpl/protocol/STPathSet.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STBase.h>
 #include <xrpl/protocol/UintTypes.h>
+
 #include <cstddef>
 #include <optional>
 

include/xrpl/protocol/STTx.h

@@ -28,6 +28,7 @@
 #include <xrpl/protocol/SecretKey.h>
 #include <xrpl/protocol/SeqProxy.h>
 #include <xrpl/protocol/TxFormats.h>
+
 #include <boost/container/flat_set.hpp>
 
 #include <functional>

include/xrpl/protocol/STValidation.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/SecretKey.h>
+
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -146,8 +147,7 @@ public:
     render() const
     {
         std::stringstream ss;
-        ss << "validation: "
-           << " ledger_hash: " << getLedgerHash()
+        ss << "validation: " << " ledger_hash: " << getLedgerHash()
            << " consensus_hash: " << getConsensusHash()
            << " sign_time: " << to_string(getSignTime())
            << " seen_time: " << to_string(getSeenTime())

include/xrpl/protocol/SecretKey.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/Seed.h>
 #include <xrpl/protocol/tokens.h>
+
 #include <array>
 #include <cstring>
 #include <string>

include/xrpl/protocol/Seed.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/Slice.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/tokens.h>
+
 #include <array>
 #include <optional>
 

include/xrpl/protocol/Serializer.h

@@ -30,6 +30,7 @@
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/protocol/HashPrefix.h>
 #include <xrpl/protocol/SField.h>
+
 #include <cstdint>
 #include <cstring>
 #include <iomanip>

include/xrpl/protocol/Sign.h

@@ -24,6 +24,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/SecretKey.h>
+
 #include <utility>
 
 namespace ripple {

include/xrpl/protocol/SystemParameters.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/chrono.h>
 #include <xrpl/protocol/XRPAmount.h>
+
 #include <cstdint>
 #include <string>
 

include/xrpl/protocol/TER.h

@@ -41,8 +41,8 @@ using TERUnderlyingType = int;
 
 enum TELcodes : TERUnderlyingType {
     // Note: Range is stable.
-    // Exact numbers are used in ripple-binary-codec:
-    //     https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json
+    // Exact numbers are used in xahau-binary-codec:
+    //     https://github.com/Xahau/xahau.js/blob/main-xahau/packages/xahau-binary-codec/src/enums/definitions.json
     // Use tokens.
 
     // -399 .. -300: L Local error (transaction fee inadequate, exceeds local
@@ -75,8 +75,8 @@ enum TELcodes : TERUnderlyingType {
 
 enum TEMcodes : TERUnderlyingType {
     // Note: Range is stable.
-    // Exact numbers are used in ripple-binary-codec:
-    //     https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json
+    // Exact numbers are used in xahau-binary-codec:
+    //     https://github.com/Xahau/xahau.js/blob/main-xahau/packages/xahau-binary-codec/src/enums/definitions.json
     // Use tokens.
 
     // -299 .. -200: M Malformed (bad signature)
@@ -151,8 +151,8 @@ enum TEMcodes : TERUnderlyingType {
 
 enum TEFcodes : TERUnderlyingType {
     // Note: Range is stable.
-    // Exact numbers are used in ripple-binary-codec:
-    //     https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json
+    // Exact numbers are used in xahau-binary-codec:
+    //     https://github.com/Xahau/xahau.js/blob/main-xahau/packages/xahau-binary-codec/src/enums/definitions.json
     // Use tokens.
 
     // -199 .. -100: F
@@ -199,8 +199,8 @@ enum TEFcodes : TERUnderlyingType {
 
 enum TERcodes : TERUnderlyingType {
     // Note: Range is stable.
-    // Exact numbers are used in ripple-binary-codec:
-    //     https://github.com/ripple/ripple-binary-codec/blob/master/src/enums/definitions.json
+    // Exact numbers are used in xahau-binary-codec:
+    //     https://github.com/Xahau/xahau.js/blob/main-xahau/packages/xahau-binary-codec/src/enums/definitions.json
     // Use tokens.
 
     // -99 .. -1: R Retry

include/xrpl/protocol/TxFlags.h

@@ -20,10 +20,10 @@
 #ifndef RIPPLE_PROTOCOL_TXFLAGS_H_INCLUDED
 #define RIPPLE_PROTOCOL_TXFLAGS_H_INCLUDED
 
-#include <cstdint>
-
 #include <xrpl/protocol/LedgerFormats.h>
 
+#include <cstdint>
+
 namespace ripple {
 
 /** Transaction flags.

include/xrpl/protocol/TxMeta.h

@@ -24,7 +24,9 @@
 #include <xrpl/protocol/STArray.h>
 #include <xrpl/protocol/STLedgerEntry.h>
 #include <xrpl/protocol/TER.h>
+
 #include <boost/container/flat_set.hpp>
+
 #include <optional>
 
 namespace ripple {

include/xrpl/protocol/detail/STVar.h

@@ -23,12 +23,9 @@
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STBase.h>
 #include <xrpl/protocol/Serializer.h>
-#include <concepts>
+
 #include <cstddef>
-#include <cstdint>
 #include <type_traits>
-#include <typeinfo>
-#include <utility>
 
 namespace ripple {
 namespace detail {

include/xrpl/protocol/detail/features.macro

@@ -23,6 +23,9 @@
 #if !defined(XRPL_FIX)
 #error "undefined macro: XRPL_FIX"
 #endif
+#if !defined(XRPL_RETIRE)
+#error "undefined macro: XRPL_RETIRE"
+#endif
 
 // Add new amendments to the top of this list.
 // Keep it sorted in reverse chronological order.
@@ -142,3 +145,22 @@ XRPL_FIX    (NFTokenDirV1,          Supported::yes, VoteBehavior::Obsolete)
 XRPL_FEATURE(NonFungibleTokensV1,   Supported::yes, VoteBehavior::Obsolete)
 XRPL_FEATURE(CryptoConditionsSuite, Supported::yes, VoteBehavior::Obsolete)
 
+// The following amendments have been active for at least two years. Their
+// pre-amendment code has been removed and the identifiers are deprecated.
+// All known amendments and amendments that may appear in a validated
+// ledger must be registered either here or above with the "active" amendments
+XRPL_RETIRE(MultiSign)
+XRPL_RETIRE(TrustSetAuth)
+XRPL_RETIRE(FeeEscalation)
+XRPL_RETIRE(PayChan)
+XRPL_RETIRE(CryptoConditions)
+XRPL_RETIRE(TickSize)
+XRPL_RETIRE(fix1368)
+XRPL_RETIRE(Escrow)
+XRPL_RETIRE(fix1373)
+XRPL_RETIRE(EnforceInvariants)
+XRPL_RETIRE(SortedDirectories)
+XRPL_RETIRE(fix1201)
+XRPL_RETIRE(fix1512)
+XRPL_RETIRE(fix1523)
+XRPL_RETIRE(fix1528)

include/xrpl/protocol/digest.h

@@ -22,7 +22,9 @@
 
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/crypto/secure_erase.h>
+
 #include <boost/endian/conversion.hpp>
+
 #include <algorithm>
 #include <array>