Prevent consensus from getting stuck in the establish phase (#5277)

- Detects if the consensus process is "stalled". If it is, then we can declare a
  consensus and end successfully even if we do not have 80% agreement on
  our proposal.
  - "Stalled" is defined as:
    - We have a close time consensus
    - Each disputed transaction is individually stalled:
      - It has been in the final "stuck" 95% requirement for at least 2
        (avMIN_ROUNDS) "inner rounds" of phaseEstablish,
      - and either all of the other trusted proposers or this validator, if proposing,
        have had the same vote(s) for at least 4 (avSTALLED_ROUNDS) "inner
        rounds", and at least 80% of the validators (including this one, if
        appropriate) agree about the vote (whether yes or no).
- If we have been in the establish phase for more than 10x the previous
  consensus establish phase's time, then consensus is considered "expired",
  and we will leave the round, which sends a partial validation (indicating
  that the node is moving on without validating). Two restrictions avoid
  prematurely exiting, or having an extended exit in extreme situations.
  - The 10x time is clamped to be within a range of 15s
    (ledgerMAX_CONSENSUS) to 120s (ledgerABANDON_CONSENSUS).
  - If consensus has not had an opportunity to walk through all avalanche
    states (defined as not going through 8 "inner rounds" of phaseEstablish),
    then ConsensusState::Expired is treated as ConsensusState::No.
- When enough nodes leave the round, any remaining nodes will see they've
  fallen behind, and move on, too, generally before hitting the timeout. Any
  validations or partial validations sent during this time will help the
  consensus process bring the nodes back together.

.clang-format

@@ -44,6 +44,7 @@ DerivePointerAlignment: false
 DisableFormat:   false
 ExperimentalAutoDetectBinPacking: false
 ForEachMacros:   [ Q_FOREACH,  BOOST_FOREACH ]
+IncludeBlocks:   Regroup
 IncludeCategories:
   - Regex:           '^<(test)/'
     Priority:        0
@@ -53,8 +54,12 @@ IncludeCategories:
     Priority:        2
   - Regex:           '^<(boost)/'
     Priority:        3
-  - Regex:           '.*'
+  - Regex:           '^.*/'
     Priority:        4
+  - Regex:           '^.*\.h'
+    Priority:        5
+  - Regex:           '.*'
+    Priority:        6
 IncludeIsMainRegex: '$'
 IndentCaseLabels: true
 IndentFunctionDeclarationAfterType: false

.github/workflows/verify-generated-headers.yml

@@ -18,26 +18,12 @@ jobs:
             generator: bash ./hook/generate_sfcodes.sh
           - target: hook/tts.h
             generator: ./hook/generate_tts.sh
-    runs-on: ubuntu-24.04
-    env:
-      CLANG_VERSION: 18
+    runs-on: ubuntu-latest
     name: ${{ matrix.target }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Install clang-format
-        run: |
-          codename=$( lsb_release --codename --short )
-          sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null <<EOF
-          deb http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
-          deb-src http://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${CLANG_VERSION} main
-          EOF
-          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add
-          sudo apt-get update
-          sudo apt-get install clang-format-${CLANG_VERSION}
-          sudo ln -sf /usr/bin/clang-format-${CLANG_VERSION} /usr/local/bin/clang-format
-
       - name: Verify ${{ matrix.target }}
         run: |
           set -euo pipefail

.mise.toml

@@ -1,2 +0,0 @@
-[tools]
-clang-format = "18"

Builds/levelization/results/loops.txt

@@ -10,9 +10,6 @@ Loop: test.jtx test.toplevel
 Loop: test.jtx test.unit_test
   test.unit_test == test.jtx
 
-Loop: xrpl.hook xrpld.app
-  xrpld.app > xrpl.hook
-
 Loop: xrpl.protocol xrpld.app
   xrpld.app > xrpl.protocol
 
@@ -32,7 +29,7 @@ Loop: xrpld.app xrpld.overlay
   xrpld.overlay ~= xrpld.app
 
 Loop: xrpld.app xrpld.peerfinder
-  xrpld.app > xrpld.peerfinder
+  xrpld.peerfinder ~= xrpld.app
 
 Loop: xrpld.app xrpld.rpc
   xrpld.rpc > xrpld.app
@@ -47,7 +44,7 @@ Loop: xrpld.core xrpld.perflog
   xrpld.perflog == xrpld.core
 
 Loop: xrpld.net xrpld.rpc
-  xrpld.rpc > xrpld.net
+  xrpld.rpc ~= xrpld.net
 
 Loop: xrpld.overlay xrpld.rpc
   xrpld.rpc ~= xrpld.overlay

Builds/levelization/results/ordering.txt

@@ -7,6 +7,7 @@ libxrpl.protocol > xrpl.hook
 libxrpl.protocol > xrpl.json
 libxrpl.protocol > xrpl.protocol
 libxrpl.resource > xrpl.basics
+libxrpl.resource > xrpl.json
 libxrpl.resource > xrpl.resource
 libxrpl.server > xrpl.basics
 libxrpl.server > xrpl.json
@@ -43,6 +44,7 @@ test.consensus > xrpld.app
 test.consensus > xrpld.consensus
 test.consensus > xrpld.core
 test.consensus > xrpld.ledger
+test.consensus > xrpl.json
 test.consensus > xrpl.protocol
 test.core > test.jtx
 test.core > test.toplevel
@@ -60,7 +62,6 @@ test.json > test.jtx
 test.json > xrpl.json
 test.jtx > xrpl.basics
 test.jtx > xrpld.app
-test.jtx > xrpld.consensus
 test.jtx > xrpld.core
 test.jtx > xrpld.ledger
 test.jtx > xrpld.net
@@ -154,6 +155,7 @@ xrpld.app > xrpl.basics
 xrpld.app > xrpld.conditions
 xrpld.app > xrpld.consensus
 xrpld.app > xrpld.perflog
+xrpld.app > xrpl.hook
 xrpld.app > xrpl.json
 xrpld.app > xrpl.resource
 xrpld.conditions > xrpl.basics
@@ -165,7 +167,6 @@ xrpld.core > xrpl.basics
 xrpld.core > xrpl.json
 xrpld.core > xrpl.protocol
 xrpld.ledger > xrpl.basics
-xrpld.ledger > xrpld.core
 xrpld.ledger > xrpl.json
 xrpld.ledger > xrpl.protocol
 xrpld.net > xrpl.basics
@@ -190,7 +191,6 @@ xrpld.peerfinder > xrpld.core
 xrpld.peerfinder > xrpl.protocol
 xrpld.perflog > xrpl.basics
 xrpld.perflog > xrpl.json
-xrpld.perflog > xrpl.protocol
 xrpld.rpc > xrpl.basics
 xrpld.rpc > xrpld.core
 xrpld.rpc > xrpld.ledger

conanfile.py

@@ -123,7 +123,7 @@ class Xrpl(ConanFile):
         if self.options.jemalloc:
             self.requires('jemalloc/5.3.0')
         if self.options.rocksdb:
-            self.requires('rocksdb/6.29.5')
+            self.requires('rocksdb/9.7.3')
 
     exports_sources = (
         'CMakeLists.txt',

examples/example/src/example.cpp

@@ -1,7 +1,7 @@
-#include <cstdio>
-
 #include <xrpl/protocol/BuildInfo.h>
 
+#include <cstdio>
+
 int main(int argc, char const** argv) {
     std::printf("%s\n", ripple::BuildInfo::getVersionString().c_str());
     return 0;

external/rocksdb/conandata.yml

@@ -1,27 +1,12 @@
 sources:
-  "6.29.5":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.29.5.tar.gz"
-    sha256: "ddbf84791f0980c0bbce3902feb93a2c7006f6f53bfd798926143e31d4d756f0"
-  "6.27.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.27.3.tar.gz"
-    sha256: "ee29901749b9132692b26f0a6c1d693f47d1a9ed8e3771e60556afe80282bf58"
-  "6.20.3":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v6.20.3.tar.gz"
-    sha256: "c6502c7aae641b7e20fafa6c2b92273d935d2b7b2707135ebd9a67b092169dca"
-  "8.8.1":
-    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v8.8.1.tar.gz"
-    sha256: "056c7e21ad8ae36b026ac3b94b9d6e0fcc60e1d937fc80330921e4181be5c36e"
+  "9.7.3":
+    url: "https://github.com/facebook/rocksdb/archive/refs/tags/v9.7.3.tar.gz"
+    sha256: "acfabb989cbfb5b5c4d23214819b059638193ec33dad2d88373c46448d16d38b"
 patches:
-  "6.29.5":
-    - patch_file: "patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
-      patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"
-    - patch_file: "patches/6.29.5-0002-exclude-thirdparty.patch"
+  "9.7.3":
+    - patch_file: "patches/9.x.x-0001-exclude-thirdparty.patch"
       patch_description: "Do not include thirdparty.inc"
       patch_type: "portability"
-  "6.27.3":
-    - patch_file: "patches/6.27.3-0001-add-include-cstdint-for-gcc-13.patch"
-      patch_description: "Fix build with gcc 13 by including cstdint"
+    - patch_file: "patches/9.7.3-0001-memory-leak.patch"
+      patch_description: "Fix a leak of obsolete blob files left open until DB::Close()"
       patch_type: "portability"
-      patch_source: "https://github.com/facebook/rocksdb/pull/11118"

external/rocksdb/conanfile.py

@@ -15,10 +15,10 @@ required_conan_version = ">=1.53.0"
 
 class RocksDBConan(ConanFile):
     name = "rocksdb"
-    homepage = "https://github.com/facebook/rocksdb"
+    description = "A library that provides an embeddable, persistent key-value store for fast storage"
     license = ("GPL-2.0-only", "Apache-2.0")
     url = "https://github.com/conan-io/conan-center-index"
-    description = "A library that provides an embeddable, persistent key-value store for fast storage"
+    homepage = "https://github.com/facebook/rocksdb"
     topics = ("database", "leveldb", "facebook", "key-value")
     package_type = "library"
     settings = "os", "arch", "compiler", "build_type"
@@ -58,12 +58,12 @@ class RocksDBConan(ConanFile):
     @property
     def _compilers_minimum_version(self):
         return {} if self._min_cppstd == "11" else {
-                "apple-clang": "10",
-                "clang": "7",
-                "gcc": "7",
-                "msvc": "191",
-                "Visual Studio": "15",
-            }
+            "apple-clang": "10",
+            "clang": "7",
+            "gcc": "7",
+            "msvc": "191",
+            "Visual Studio": "15",
+        }
 
     def export_sources(self):
         export_conandata_patches(self)
@@ -115,9 +115,9 @@ class RocksDBConan(ConanFile):
         check_min_vs(self, "191")
 
         if self.version == "6.20.3" and \
-           self.settings.os == "Linux" and \
-           self.settings.compiler == "gcc" and \
-           Version(self.settings.compiler.version) < "5":
+                self.settings.os == "Linux" and \
+                self.settings.compiler == "gcc" and \
+                Version(self.settings.compiler.version) < "5":
             raise ConanInvalidConfiguration("Rocksdb 6.20.3 is not compilable with gcc <5.") # See https://github.com/facebook/rocksdb/issues/3522
 
     def source(self):
@@ -163,6 +163,8 @@ class RocksDBConan(ConanFile):
         if self.options.with_jemalloc:
             deps.set_property("jemalloc", "cmake_file_name", "JeMalloc")
             deps.set_property("jemalloc", "cmake_target_name", "JeMalloc::JeMalloc")
+        if self.options.with_zstd:
+            deps.set_property("zstd", "cmake_target_name", "zstd::zstd")
         deps.generate()
 
     def build(self):

external/rocksdb/patches/6.29.5-0001-add-include-cstdint-for-gcc-13.patch

@@ -1,30 +0,0 @@
---- a/include/rocksdb/utilities/checkpoint.h
-+++ b/include/rocksdb/utilities/checkpoint.h
-@@ -8,6 +8,7 @@
- #pragma once
- #ifndef ROCKSDB_LITE
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- #include "rocksdb/status.h"
---- a/table/block_based/data_block_hash_index.h
-+++ b/table/block_based/data_block_hash_index.h
-@@ -5,6 +5,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <string>
- #include <vector>
- 
---- a/util/string_util.h
-+++ b/util/string_util.h
-@@ -6,6 +6,7 @@
- 
- #pragma once
- 
-+#include <cstdint>
- #include <sstream>
- #include <string>
- #include <unordered_map>

external/rocksdb/patches/6.29.5-0002-exclude-thirdparty.patch

@@ -1,16 +0,0 @@
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index ec59d4491..35577c998 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -101 +100,0 @@ if(MSVC)
--  option(WITH_GFLAGS "build with GFlags" OFF)
-@@ -103,2 +102,2 @@ if(MSVC)
--  include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
--else()
-+endif()
-+
-@@ -117 +116 @@ else()
--  if(MINGW)
-+  if(MINGW OR MSVC)
-@@ -183 +181,0 @@ else()
--endif()

external/rocksdb/patches/9.7.3-0001-memory-leak.patch

@@ -0,0 +1,319 @@
+diff --git a/HISTORY.md b/HISTORY.md
+index 36d472229..05ad1a202 100644
+--- a/HISTORY.md
++++ b/HISTORY.md
+@@ -1,6 +1,10 @@
+ # Rocksdb Change Log
+ > NOTE: Entries for next release do not go here. Follow instructions in `unreleased_history/README.txt`
+ 
++## 9.7.4 (10/31/2024)
++### Bug Fixes
++* Fix a leak of obsolete blob files left open until DB::Close(). This bug was introduced in version 9.4.0.
++
+ ## 9.7.3 (10/16/2024)
+ ### Behavior Changes
+ * OPTIONS file to be loaded by remote worker is now preserved so that it does not get purged by the primary host. A similar technique as how we are preserving new SST files from getting purged is used for this. min_options_file_numbers_ is tracked like pending_outputs_ is tracked.
+diff --git a/db/blob/blob_file_cache.cc b/db/blob/blob_file_cache.cc
+index 5f340aadf..1b9faa238 100644
+--- a/db/blob/blob_file_cache.cc
++++ b/db/blob/blob_file_cache.cc
+@@ -42,6 +42,7 @@ Status BlobFileCache::GetBlobFileReader(
+   assert(blob_file_reader);
+   assert(blob_file_reader->IsEmpty());
+ 
++  // NOTE: sharing same Cache with table_cache
+   const Slice key = GetSliceForKey(&blob_file_number);
+ 
+   assert(cache_);
+@@ -98,4 +99,13 @@ Status BlobFileCache::GetBlobFileReader(
+   return Status::OK();
+ }
+ 
++void BlobFileCache::Evict(uint64_t blob_file_number) {
++  // NOTE: sharing same Cache with table_cache
++  const Slice key = GetSliceForKey(&blob_file_number);
++
++  assert(cache_);
++
++  cache_.get()->Erase(key);
++}
++
+ }  // namespace ROCKSDB_NAMESPACE
+diff --git a/db/blob/blob_file_cache.h b/db/blob/blob_file_cache.h
+index 740e67ada..6858d012b 100644
+--- a/db/blob/blob_file_cache.h
++++ b/db/blob/blob_file_cache.h
+@@ -36,6 +36,15 @@ class BlobFileCache {
+                            uint64_t blob_file_number,
+                            CacheHandleGuard<BlobFileReader>* blob_file_reader);
+ 
++  // Called when a blob file is obsolete to ensure it is removed from the cache
++  // to avoid effectively leaking the open file and assicated memory
++  void Evict(uint64_t blob_file_number);
++
++  // Used to identify cache entries for blob files (not normally useful)
++  static const Cache::CacheItemHelper* GetHelper() {
++    return CacheInterface::GetBasicHelper();
++  }
++
+  private:
+   using CacheInterface =
+       BasicTypedCacheInterface<BlobFileReader, CacheEntryRole::kMisc>;
+diff --git a/db/column_family.h b/db/column_family.h
+index e4b7adde8..86637736a 100644
+--- a/db/column_family.h
++++ b/db/column_family.h
+@@ -401,6 +401,7 @@ class ColumnFamilyData {
+                          SequenceNumber earliest_seq);
+ 
+   TableCache* table_cache() const { return table_cache_.get(); }
++  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
+   BlobSource* blob_source() const { return blob_source_.get(); }
+ 
+   // See documentation in compaction_picker.h
+diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc
+index 261593423..06573ac2e 100644
+--- a/db/db_impl/db_impl.cc
++++ b/db/db_impl/db_impl.cc
+@@ -659,8 +659,9 @@ Status DBImpl::CloseHelper() {
+   // We need to release them before the block cache is destroyed. The block
+   // cache may be destroyed inside versions_.reset(), when column family data
+   // list is destroyed, so leaving handles in table cache after
+-  // versions_.reset() may cause issues.
+-  // Here we clean all unreferenced handles in table cache.
++  // versions_.reset() may cause issues. Here we clean all unreferenced handles
++  // in table cache, and (for certain builds/conditions) assert that no obsolete
++  // files are hanging around unreferenced (leak) in the table/blob file cache.
+   // Now we assume all user queries have finished, so only version set itself
+   // can possibly hold the blocks from block cache. After releasing unreferenced
+   // handles here, only handles held by version set left and inside
+@@ -668,6 +669,9 @@ Status DBImpl::CloseHelper() {
+   // time a handle is released, we erase it from the cache too. By doing that,
+   // we can guarantee that after versions_.reset(), table cache is empty
+   // so the cache can be safely destroyed.
++#ifndef NDEBUG
++  TEST_VerifyNoObsoleteFilesCached(/*db_mutex_already_held=*/true);
++#endif  // !NDEBUG
+   table_cache_->EraseUnRefEntries();
+ 
+   for (auto& txn_entry : recovered_transactions_) {
+@@ -3227,6 +3231,8 @@ Status DBImpl::MultiGetImpl(
+       s = Status::Aborted();
+       break;
+     }
++    // This could be a long-running operation
++    ROCKSDB_THREAD_YIELD_HOOK();
+   }
+ 
+   // Post processing (decrement reference counts and record statistics)
+diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h
+index 5e4fa310b..ccc0abfa7 100644
+--- a/db/db_impl/db_impl.h
++++ b/db/db_impl/db_impl.h
+@@ -1241,9 +1241,14 @@ class DBImpl : public DB {
+   static Status TEST_ValidateOptions(const DBOptions& db_options) {
+     return ValidateOptions(db_options);
+   }
+-
+ #endif  // NDEBUG
+ 
++  // In certain configurations, verify that the table/blob file cache only
++  // contains entries for live files, to check for effective leaks of open
++  // files. This can only be called when purging of obsolete files has
++  // "settled," such as during parts of DB Close().
++  void TEST_VerifyNoObsoleteFilesCached(bool db_mutex_already_held) const;
++
+   // persist stats to column family "_persistent_stats"
+   void PersistStats();
+ 
+diff --git a/db/db_impl/db_impl_debug.cc b/db/db_impl/db_impl_debug.cc
+index 790a50d7a..67f5b4aaf 100644
+--- a/db/db_impl/db_impl_debug.cc
++++ b/db/db_impl/db_impl_debug.cc
+@@ -9,6 +9,7 @@
+ 
+ #ifndef NDEBUG
+ 
++#include "db/blob/blob_file_cache.h"
+ #include "db/column_family.h"
+ #include "db/db_impl/db_impl.h"
+ #include "db/error_handler.h"
+@@ -328,5 +329,49 @@ size_t DBImpl::TEST_EstimateInMemoryStatsHistorySize() const {
+   InstrumentedMutexLock l(&const_cast<DBImpl*>(this)->stats_history_mutex_);
+   return EstimateInMemoryStatsHistorySize();
+ }
++
++void DBImpl::TEST_VerifyNoObsoleteFilesCached(
++    bool db_mutex_already_held) const {
++  // This check is somewhat expensive and obscure to make a part of every
++  // unit test in every build variety. Thus, we only enable it for ASAN builds.
++  if (!kMustFreeHeapAllocations) {
++    return;
++  }
++
++  std::optional<InstrumentedMutexLock> l;
++  if (db_mutex_already_held) {
++    mutex_.AssertHeld();
++  } else {
++    l.emplace(&mutex_);
++  }
++
++  std::vector<uint64_t> live_files;
++  for (auto cfd : *versions_->GetColumnFamilySet()) {
++    if (cfd->IsDropped()) {
++      continue;
++    }
++    // Sneakily add both SST and blob files to the same list
++    cfd->current()->AddLiveFiles(&live_files, &live_files);
++  }
++  std::sort(live_files.begin(), live_files.end());
++
++  auto fn = [&live_files](const Slice& key, Cache::ObjectPtr, size_t,
++                          const Cache::CacheItemHelper* helper) {
++    if (helper != BlobFileCache::GetHelper()) {
++      // Skip non-blob files for now
++      // FIXME: diagnose and fix the leaks of obsolete SST files revealed in
++      // unit tests.
++      return;
++    }
++    // See TableCache and BlobFileCache
++    assert(key.size() == sizeof(uint64_t));
++    uint64_t file_number;
++    GetUnaligned(reinterpret_cast<const uint64_t*>(key.data()), &file_number);
++    // Assert file is in sorted live_files
++    assert(
++        std::binary_search(live_files.begin(), live_files.end(), file_number));
++  };
++  table_cache_->ApplyToAllEntries(fn, {});
++}
+ }  // namespace ROCKSDB_NAMESPACE
+ #endif  // NDEBUG
+diff --git a/db/db_iter.cc b/db/db_iter.cc
+index e02586377..bf4749eb9 100644
+--- a/db/db_iter.cc
++++ b/db/db_iter.cc
+@@ -540,6 +540,8 @@ bool DBIter::FindNextUserEntryInternal(bool skipping_saved_key,
+     } else {
+       iter_.Next();
+     }
++    // This could be a long-running operation due to tombstones, etc.
++    ROCKSDB_THREAD_YIELD_HOOK();
+   } while (iter_.Valid());
+ 
+   valid_ = false;
+diff --git a/db/table_cache.cc b/db/table_cache.cc
+index 71fc29c32..8a5be75e8 100644
+--- a/db/table_cache.cc
++++ b/db/table_cache.cc
+@@ -164,6 +164,7 @@ Status TableCache::GetTableReader(
+ }
+ 
+ Cache::Handle* TableCache::Lookup(Cache* cache, uint64_t file_number) {
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&file_number);
+   return cache->Lookup(key);
+ }
+@@ -179,6 +180,7 @@ Status TableCache::FindTable(
+     size_t max_file_size_for_l0_meta_pin, Temperature file_temperature) {
+   PERF_TIMER_GUARD_WITH_CLOCK(find_table_nanos, ioptions_.clock);
+   uint64_t number = file_meta.fd.GetNumber();
++  // NOTE: sharing same Cache with BlobFileCache
+   Slice key = GetSliceForFileNumber(&number);
+   *handle = cache_.Lookup(key);
+   TEST_SYNC_POINT_CALLBACK("TableCache::FindTable:0",
+diff --git a/db/version_builder.cc b/db/version_builder.cc
+index ed8ab8214..c98f53f42 100644
+--- a/db/version_builder.cc
++++ b/db/version_builder.cc
+@@ -24,6 +24,7 @@
+ #include <vector>
+ 
+ #include "cache/cache_reservation_manager.h"
++#include "db/blob/blob_file_cache.h"
+ #include "db/blob/blob_file_meta.h"
+ #include "db/dbformat.h"
+ #include "db/internal_stats.h"
+@@ -744,12 +745,9 @@ class VersionBuilder::Rep {
+       return Status::Corruption("VersionBuilder", oss.str());
+     }
+ 
+-    // Note: we use C++11 for now but in C++14, this could be done in a more
+-    // elegant way using generalized lambda capture.
+-    VersionSet* const vs = version_set_;
+-    const ImmutableCFOptions* const ioptions = ioptions_;
+-
+-    auto deleter = [vs, ioptions](SharedBlobFileMetaData* shared_meta) {
++    auto deleter = [vs = version_set_, ioptions = ioptions_,
++                    bc = cfd_ ? cfd_->blob_file_cache()
++                              : nullptr](SharedBlobFileMetaData* shared_meta) {
+       if (vs) {
+         assert(ioptions);
+         assert(!ioptions->cf_paths.empty());
+@@ -758,6 +756,9 @@ class VersionBuilder::Rep {
+         vs->AddObsoleteBlobFile(shared_meta->GetBlobFileNumber(),
+                                 ioptions->cf_paths.front().path);
+       }
++      if (bc) {
++        bc->Evict(shared_meta->GetBlobFileNumber());
++      }
+ 
+       delete shared_meta;
+     };
+@@ -766,7 +767,7 @@ class VersionBuilder::Rep {
+         blob_file_number, blob_file_addition.GetTotalBlobCount(),
+         blob_file_addition.GetTotalBlobBytes(),
+         blob_file_addition.GetChecksumMethod(),
+-        blob_file_addition.GetChecksumValue(), deleter);
++        blob_file_addition.GetChecksumValue(), std::move(deleter));
+ 
+     mutable_blob_file_metas_.emplace(
+         blob_file_number, MutableBlobFileMetaData(std::move(shared_meta)));
+diff --git a/db/version_set.h b/db/version_set.h
+index 9336782b1..024f869e7 100644
+--- a/db/version_set.h
++++ b/db/version_set.h
+@@ -1514,7 +1514,6 @@ class VersionSet {
+   void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+ 
+   void AddObsoleteBlobFile(uint64_t blob_file_number, std::string path) {
+-    // TODO: Erase file from BlobFileCache?
+     obsolete_blob_files_.emplace_back(blob_file_number, std::move(path));
+   }
+ 
+diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h
+index 2a19796b8..0afa2cab1 100644
+--- a/include/rocksdb/version.h
++++ b/include/rocksdb/version.h
+@@ -13,7 +13,7 @@
+ // minor or major version number planned for release.
+ #define ROCKSDB_MAJOR 9
+ #define ROCKSDB_MINOR 7
+-#define ROCKSDB_PATCH 3
++#define ROCKSDB_PATCH 4
+ 
+ // Do not use these. We made the mistake of declaring macros starting with
+ // double underscore. Now we have to live with our choice. We'll deprecate these
+diff --git a/port/port.h b/port/port.h
+index 13aa56d47..141716e5b 100644
+--- a/port/port.h
++++ b/port/port.h
+@@ -19,3 +19,19 @@
+ #elif defined(OS_WIN)
+ #include "port/win/port_win.h"
+ #endif
++
++#ifdef OS_LINUX
++// A temporary hook into long-running RocksDB threads to support modifying their
++// priority etc. This should become a public API hook once the requirements
++// are better understood.
++extern "C" void RocksDbThreadYield() __attribute__((__weak__));
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {                                 \
++    if (RocksDbThreadYield) {       \
++      RocksDbThreadYield();         \
++    }                               \
++  }
++#else
++#define ROCKSDB_THREAD_YIELD_HOOK() \
++  {}
++#endif

external/rocksdb/patches/9.x.x-0001-exclude-thirdparty.patch

@@ -0,0 +1,30 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 93b884d..b715cb6 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -106,14 +106,9 @@ endif()
+ include(CMakeDependentOption)
+
+ if(MSVC)
+-  option(WITH_GFLAGS "build with GFlags" OFF)
+   option(WITH_XPRESS "build with windows built in compression" OFF)
+-  option(ROCKSDB_SKIP_THIRDPARTY "skip thirdparty.inc" OFF)
+-
+-  if(NOT ROCKSDB_SKIP_THIRDPARTY)
+-    include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
+-  endif()
+-else()
++endif()
++if(TRUE)
+   if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
+     # FreeBSD has jemalloc as default malloc
+     # but it does not have all the jemalloc files in include/...
+@@ -126,7 +121,7 @@ else()
+     endif()
+   endif()
+
+-  if(MINGW)
++  if(MSVC OR MINGW)
+     option(WITH_GFLAGS "build with GFlags" OFF)
+   else()
+     option(WITH_GFLAGS "build with GFlags" ON)

hook/extern.h

@@ -82,7 +82,7 @@ sto_erase(
     uint32_t field_id);
 
 extern int64_t
-etxn_burden();
+etxn_burden(void);
 
 extern int64_t
 etxn_details(uint32_t write_ptr, uint32_t write_len);
@@ -94,7 +94,7 @@ extern int64_t
 etxn_reserve(uint32_t count);
 
 extern int64_t
-etxn_generation();
+etxn_generation(void);
 
 extern int64_t
 etxn_nonce(uint32_t write_ptr, uint32_t write_len);
@@ -149,7 +149,7 @@ extern int64_t
 float_divide(int64_t float1, int64_t float2);
 
 extern int64_t
-float_one();
+float_one(void);
 
 extern int64_t
 float_mantissa(int64_t float1);
@@ -167,13 +167,13 @@ extern int64_t
 float_root(int64_t float1, uint32_t n);
 
 extern int64_t
-fee_base();
+fee_base(void);
 
 extern int64_t
-ledger_seq();
+ledger_seq(void);
 
 extern int64_t
-ledger_last_time();
+ledger_last_time(void);
 
 extern int64_t
 ledger_last_hash(uint32_t write_ptr, uint32_t write_len);
@@ -213,13 +213,13 @@ hook_param(
     uint32_t read_len);
 
 extern int64_t
-hook_again();
+hook_again(void);
 
 extern int64_t
 hook_skip(uint32_t read_ptr, uint32_t read_len, uint32_t flags);
 
 extern int64_t
-hook_pos();
+hook_pos(void);
 
 extern int64_t
 slot(uint32_t write_ptr, uint32_t write_len, uint32_t slot);
@@ -299,19 +299,19 @@ extern int64_t
 trace_float(uint32_t read_ptr, uint32_t read_len, int64_t float1);
 
 extern int64_t
-otxn_burden();
+otxn_burden(void);
 
 extern int64_t
 otxn_field(uint32_t write_ptr, uint32_t write_len, uint32_t field_id);
 
 extern int64_t
-otxn_generation();
+otxn_generation(void);
 
 extern int64_t
 otxn_id(uint32_t write_ptr, uint32_t write_len, uint32_t flags);
 
 extern int64_t
-otxn_type();
+otxn_type(void);
 
 extern int64_t
 otxn_slot(uint32_t slot_no);

hook/generate_extern.sh

@@ -4,7 +4,7 @@ set -eu
 SCRIPT_DIR=$(dirname "$0")
 SCRIPT_DIR=$(cd "$SCRIPT_DIR" && pwd)
 
-APPLY_HOOK="$SCRIPT_DIR/../include/xrpl/hook/hook_api.macro"
+APPLY_HOOK="$SCRIPT_DIR/../src/xrpld/app/hook/applyHook.h"
 
 {
     echo '// For documentation please see: https://xrpl-hooks.readme.io/reference/'
@@ -19,36 +19,127 @@ APPLY_HOOK="$SCRIPT_DIR/../include/xrpl/hook/hook_api.macro"
             return s;
         }
 
+        function emit(ret, name, argc, argt, argn) {
+            attr = (name == "_g") ? " __attribute__((noduplicate))" : "";
+            if (!first)
+                printf("\n");
+            first = 0;
+            printf("extern %s%s\n", ret, attr);
+            if (argc == 0) {
+                printf("%s(void);\n", name);
+                return;
+            }
+            if (argc <= 3) {
+                line = argt[1] " " argn[1];
+                for (i = 2; i <= argc; ++i)
+                    line = line ", " argt[i] " " argn[i];
+                printf("%s(%s);\n", name, line);
+                return;
+            }
+            printf("%s(\n", name);
+            for (i = 1; i <= argc; ++i) {
+                sep = (i < argc) ? "," : ");";
+                printf("    %s %s%s\n", argt[i], argn[i], sep);
+            }
+        }
+
+        function process(buffer, kind,    payload, parts, n, i, arg, tokens, argc, argt, argn) {
+            if (kind == "func")
+                sub(/^DECLARE_HOOK_FUNCTION[[:space:]]*\(/, "", buffer);
+            else
+                sub(/^DECLARE_HOOK_FUNCNARG[[:space:]]*\(/, "", buffer);
+            buffer = trim(buffer);
+            sub(/\)[[:space:]]*$/, "", buffer);
+            n = split(buffer, parts, ",");
+            for (i = 1; i <= n; ++i)
+                parts[i] = trim(parts[i]);
+            ret = parts[1];
+            name = parts[2];
+            argc = 0;
+            delete argt;
+            delete argn;
+            for (i = 3; i <= n; ++i) {
+                arg = parts[i];
+                if (arg == "")
+                    continue;
+                split(arg, tokens, /[[:space:]]+/);
+                if (length(tokens) < 2)
+                    continue;
+                ++argc;
+                argt[argc] = tokens[1];
+                argn[argc] = tokens[2];
+            }
+            emit(ret, name, argc, argt, argn);
+        }
+
+        BEGIN {
+            first = 1;
+            in_block = 0;
+            in_macro = 0;
+        }
+
         {
             line = $0;
-            
-            # Skip block comments
-            if (line ~ /\/\*/) {
+            if (in_block) {
+                if (line ~ /\*\//) {
+                    sub(/.*\*\//, "", line);
+                    in_block = 0;
+                }
+                else
+                    next;
+            }
+            while (line ~ /\/\*/) {
+                if (line ~ /\/\*.*\*\//) {
+                    gsub(/\/\*.*\*\//, "", line);
+                }
+                else {
+                    sub(/\/\*.*/, "", line);
+                    in_block = 1;
+                    break;
+                }
+            }
+            sub(/\/\/.*$/, "", line);
+            line = trim(line);
+            if (line == "")
+                next;
+
+            if (!in_macro && line ~ /^DECLARE_HOOK_FUNCTION\(/) {
+                buffer = line;
+                kind = "func";
+                if (line ~ /\);[[:space:]]*$/) {
+                    sub(/\);[[:space:]]*$/, "", buffer);
+                    process(buffer, kind);
+                }
+                else
+                    in_macro = 1;
                 next;
             }
-            
-            # Look for comment lines that start with // and contain function signature
-            if (line ~ /^[[:space:]]*\/\/[[:space:]]*[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]+[a-zA-Z_][a-zA-Z0-9_]*[[:space:]]*\(/) {
-                # Remove leading // and trim
-                sub(/^[[:space:]]*\/\/[[:space:]]*/, "", line);
-                line = trim(line);
-                
-                # Check if function name is "_g" to add attribute
-                if (line ~ /[[:space:]]+_g[[:space:]]*\(/) {
-                    # Insert __attribute__((noduplicate)) before _g
-                    sub(/[[:space:]]+_g/, " __attribute__((noduplicate)) _g", line);
+            if (!in_macro && line ~ /^DECLARE_HOOK_FUNCNARG\(/) {
+                buffer = line;
+                kind = "narg";
+                if (line ~ /\);[[:space:]]*$/) {
+                    sub(/\);[[:space:]]*$/, "", buffer);
+                    process(buffer, kind);
                 }
-                
-                # printf("\n");
-                
-                printf("extern %s\n\n", line);
+                else
+                    in_macro = 1;
+                next;
             }
+            if (in_macro) {
+                buffer = buffer " " line;
+                if (line ~ /\);[[:space:]]*$/) {
+                    sub(/\);[[:space:]]*$/, "", buffer);
+                    process(buffer, kind);
+                    in_macro = 0;
+                }
+            }
+        }
+
+        END {
+            printf("\n");
         }
     ' "$APPLY_HOOK"
 
     echo '#define HOOK_EXTERN'
     echo '#endif  // HOOK_EXTERN'
-} | (
-    cd "$SCRIPT_DIR/.."
-    clang-format --style=file -
-)
+}

hook/hookapi.h

@@ -45,8 +45,8 @@
 
 #include "error.h"
 #include "extern.h"
-#include "sfcodes.h"
 #include "macro.h"
+#include "sfcodes.h"
 #include "tts.h"
 
 #endif

include/xrpl/basics/BasicConfig.h

@@ -27,7 +27,6 @@
 
 #include <algorithm>
 #include <optional>
-#include <ostream>
 #include <string>
 #include <unordered_map>
 #include <vector>

include/xrpl/basics/Buffer.h

@@ -22,10 +22,9 @@
 
 #include <xrpl/basics/Slice.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <cstdint>
 #include <cstring>
-#include <memory>
-#include <utility>
 
 namespace ripple {
 

include/xrpl/basics/CompressionAlgorithms.h

@@ -21,9 +21,11 @@
 #define RIPPLED_COMPRESSIONALGORITHMS_H_INCLUDED
 
 #include <xrpl/basics/contract.h>
+
+#include <lz4.h>
+
 #include <algorithm>
 #include <cstdint>
-#include <lz4.h>
 #include <stdexcept>
 #include <vector>
 

include/xrpl/basics/CountedObject.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_COUNTEDOBJECT_H_INCLUDED
 
 #include <xrpl/beast/type_name.h>
+
 #include <atomic>
 #include <string>
 #include <utility>

include/xrpl/basics/Expected.h

@@ -24,9 +24,7 @@
 
 #include <boost/outcome.hpp>
 
-#include <concepts>
 #include <stdexcept>
-#include <type_traits>
 
 namespace ripple {
 
@@ -138,14 +136,14 @@ public:
     template <typename U>
         requires std::convertible_to<U, T>
     constexpr Expected(U&& r)
-        : Base(boost::outcome_v2::success(T(std::forward<U>(r))))
+        : Base(boost::outcome_v2::in_place_type_t<T>{}, std::forward<U>(r))
     {
     }
 
     template <typename U>
         requires std::convertible_to<U, E> && (!std::is_reference_v<U>)
     constexpr Expected(Unexpected<U> e)
-        : Base(boost::outcome_v2::failure(E(std::move(e.value()))))
+        : Base(boost::outcome_v2::in_place_type_t<E>{}, std::move(e.value()))
     {
     }
 
@@ -230,8 +228,7 @@ public:
 
     template <typename U>
         requires std::convertible_to<U, E> && (!std::is_reference_v<U>)
-    constexpr Expected(Unexpected<U> e)
-        : Base(boost::outcome_v2::failure(E(std::move(e.value()))))
+    constexpr Expected(Unexpected<U> e) : Base(E(std::move(e.value())))
     {
     }
 

include/xrpl/basics/LocalValue.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_LOCALVALUE_H_INCLUDED
 
 #include <boost/thread/tss.hpp>
+
 #include <memory>
 #include <unordered_map>
 

include/xrpl/basics/Log.h

@@ -22,8 +22,10 @@
 
 #include <xrpl/basics/UnorderedContainers.h>
 #include <xrpl/beast/utility/Journal.h>
+
 #include <boost/beast/core/string.hpp>
 #include <boost/filesystem.hpp>
+
 #include <fstream>
 #include <map>
 #include <memory>

include/xrpl/basics/Resolver.h

@@ -20,11 +20,11 @@
 #ifndef RIPPLE_BASICS_RESOLVER_H_INCLUDED
 #define RIPPLE_BASICS_RESOLVER_H_INCLUDED
 
+#include <xrpl/beast/net/IPEndpoint.h>
+
 #include <functional>
 #include <vector>
 
-#include <xrpl/beast/net/IPEndpoint.h>
-
 namespace ripple {
 
 class Resolver

include/xrpl/basics/ResolverAsio.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/Resolver.h>
 #include <xrpl/beast/utility/Journal.h>
+
 #include <boost/asio/io_service.hpp>
 
 namespace ripple {

include/xrpl/basics/Slice.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <algorithm>
 #include <array>
 #include <cstdint>

include/xrpl/basics/TaggedCache.h

@@ -25,6 +25,7 @@
 #include <xrpl/basics/hardened_hash.h>
 #include <xrpl/beast/clock/abstract_clock.h>
 #include <xrpl/beast/insight/Insight.h>
+
 #include <atomic>
 #include <functional>
 #include <mutex>

include/xrpl/basics/UnorderedContainers.h

@@ -25,6 +25,7 @@
 #include <xrpl/beast/hash/hash_append.h>
 #include <xrpl/beast/hash/uhash.h>
 #include <xrpl/beast/hash/xxhasher.h>
+
 #include <unordered_map>
 #include <unordered_set>
 

include/xrpl/basics/base_uint.h

@@ -33,12 +33,13 @@
 #include <xrpl/basics/strHex.h>
 #include <xrpl/beast/utility/Zero.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/endian/conversion.hpp>
 #include <boost/functional/hash.hpp>
+
 #include <algorithm>
 #include <array>
 #include <cstring>
-#include <functional>
 #include <type_traits>
 
 namespace ripple {

include/xrpl/basics/chrono.h

@@ -20,17 +20,16 @@
 #ifndef RIPPLE_BASICS_CHRONO_H_INCLUDED
 #define RIPPLE_BASICS_CHRONO_H_INCLUDED
 
-#include <date/date.h>
-
 #include <xrpl/beast/clock/abstract_clock.h>
 #include <xrpl/beast/clock/basic_seconds_clock.h>
 #include <xrpl/beast/clock/manual_clock.h>
 
+#include <date/date.h>
+
 #include <chrono>
 #include <cstdint>
 #include <ratio>
 #include <string>
-#include <type_traits>
 
 namespace ripple {
 

include/xrpl/basics/contract.h

@@ -21,9 +21,9 @@
 #define RIPPLE_BASICS_CONTRACT_H_INCLUDED
 
 #include <xrpl/beast/type_name.h>
+
 #include <exception>
 #include <string>
-#include <typeinfo>
 #include <utility>
 
 namespace ripple {

include/xrpl/basics/make_SSLContext.h

@@ -21,6 +21,7 @@
 #define RIPPLE_BASICS_MAKE_SSLCONTEXT_H_INCLUDED
 
 #include <boost/asio/ssl/context.hpp>
+
 #include <string>
 
 namespace ripple {

include/xrpl/basics/partitioned_unordered_map.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/beast/hash/uhash.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <functional>
 #include <optional>
 #include <string>

include/xrpl/basics/random.h

@@ -22,9 +22,9 @@
 
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/beast/xor_shift_engine.h>
+
 #include <cstddef>
 #include <cstdint>
-#include <cstring>
 #include <limits>
 #include <mutex>
 #include <random>

include/xrpl/basics/spinlock.h

@@ -19,6 +19,7 @@
 #define RIPPLE_BASICS_SPINLOCK_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <atomic>
 #include <limits>
 #include <type_traits>

include/xrpl/basics/tagged_integer.h

@@ -21,7 +21,9 @@
 #define BEAST_UTILITY_TAGGED_INTEGER_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <boost/operators.hpp>
+
 #include <functional>
 #include <iostream>
 #include <type_traits>

include/xrpl/beast/asio/io_latency_probe.h

@@ -21,6 +21,7 @@
 #define BEAST_ASIO_IO_LATENCY_PROBE_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/basic_waitable_timer.hpp>
 #include <boost/asio/io_service.hpp>
 

include/xrpl/beast/container/detail/aged_ordered_container.h

@@ -25,9 +25,11 @@
 #include <xrpl/beast/container/detail/aged_associative_container.h>
 #include <xrpl/beast/container/detail/aged_container_iterator.h>
 #include <xrpl/beast/container/detail/empty_base_optimization.h>
+
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/set.hpp>
 #include <boost/version.hpp>
+
 #include <algorithm>
 #include <functional>
 #include <initializer_list>

include/xrpl/beast/container/detail/aged_unordered_container.h

@@ -25,8 +25,10 @@
 #include <xrpl/beast/container/detail/aged_associative_container.h>
 #include <xrpl/beast/container/detail/aged_container_iterator.h>
 #include <xrpl/beast/container/detail/empty_base_optimization.h>
+
 #include <boost/intrusive/list.hpp>
 #include <boost/intrusive/unordered_set.hpp>
+
 #include <algorithm>
 #include <cmath>
 #include <functional>

include/xrpl/beast/container/detail/empty_base_optimization.h

@@ -11,6 +11,7 @@
 #define BEAST_CONTAINER_DETAIL_EMPTY_BASE_OPTIMIZATION_H_INCLUDED
 
 #include <boost/type_traits/is_final.hpp>
+
 #include <type_traits>
 #include <utility>
 

include/xrpl/beast/core/LexicalCast.h

@@ -23,6 +23,7 @@
 #include <xrpl/beast/utility/instrumentation.h>
 
 #include <boost/core/detail/string_view.hpp>
+
 #include <algorithm>
 #include <cerrno>
 #include <charconv>

include/xrpl/beast/core/List.h

@@ -21,7 +21,6 @@
 #define BEAST_INTRUSIVE_LIST_H_INCLUDED
 
 #include <iterator>
-#include <type_traits>
 
 namespace beast {
 

include/xrpl/beast/hash/hash_append.h

@@ -23,6 +23,7 @@
 
 #include <boost/container/flat_set.hpp>
 #include <boost/endian/conversion.hpp>
+
 #include <array>
 #include <chrono>
 #include <cstdint>

include/xrpl/beast/hash/xxhasher.h

@@ -21,6 +21,7 @@
 #define BEAST_HASH_XXHASHER_H_INCLUDED
 
 #include <boost/endian/conversion.hpp>
+
 #include <xxhash.h>
 
 #include <cstddef>

include/xrpl/beast/insight/Meter.h

@@ -20,10 +20,10 @@
 #ifndef BEAST_INSIGHT_METER_H_INCLUDED
 #define BEAST_INSIGHT_METER_H_INCLUDED
 
-#include <memory>
-
 #include <xrpl/beast/insight/MeterImpl.h>
 
+#include <memory>
+
 namespace beast {
 namespace insight {
 

include/xrpl/beast/insight/StatsDCollector.h

@@ -21,7 +21,6 @@
 #define BEAST_INSIGHT_STATSDCOLLECTOR_H_INCLUDED
 
 #include <xrpl/beast/insight/Collector.h>
-
 #include <xrpl/beast/net/IPEndpoint.h>
 #include <xrpl/beast/utility/Journal.h>
 

include/xrpl/beast/net/IPAddress.h

@@ -25,8 +25,10 @@
 #include <xrpl/beast/net/IPAddressV4.h>
 #include <xrpl/beast/net/IPAddressV6.h>
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/ip/address.hpp>
 #include <boost/functional/hash.hpp>
+
 #include <cstdint>
 #include <ios>
 #include <sstream>

include/xrpl/beast/net/IPAddressConversion.h

@@ -22,8 +22,6 @@
 
 #include <xrpl/beast/net/IPEndpoint.h>
 
-#include <sstream>
-
 #include <boost/asio.hpp>
 
 namespace beast {

include/xrpl/beast/net/IPAddressV4.h

@@ -21,7 +21,9 @@
 #define BEAST_NET_IPADDRESSV4_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <boost/asio/ip/address_v4.hpp>
+
 #include <cstdint>
 #include <functional>
 #include <ios>

include/xrpl/beast/net/IPAddressV6.h

@@ -21,7 +21,9 @@
 #define BEAST_NET_IPADDRESSV6_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <boost/asio/ip/address_v6.hpp>
+
 #include <cstdint>
 #include <functional>
 #include <ios>

include/xrpl/beast/rfc2616.h

@@ -20,10 +20,6 @@
 #ifndef BEAST_RFC2616_HPP
 #define BEAST_RFC2616_HPP
 
-// TODO: This include <cstdint> is a workaround for beast compilation bug.
-// Remove when fix https://github.com/boostorg/beast/pull/2682/ is available.
-#include <cstdint>
-
 #include <boost/beast/http/message.hpp>
 #include <boost/beast/http/rfc7230.hpp>
 #include <boost/range/algorithm/equal.hpp>
@@ -32,6 +28,7 @@
 
 #include <algorithm>
 #include <cctype>
+#include <cstdint>
 #include <iterator>
 #include <string>
 #include <utility>

include/xrpl/beast/test/yield_to.h

@@ -11,6 +11,7 @@
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/spawn.hpp>
 #include <boost/optional.hpp>
+
 #include <condition_variable>
 #include <functional>
 #include <mutex>

include/xrpl/beast/unit_test/match.h

@@ -9,6 +9,7 @@
 #define BEAST_UNIT_TEST_MATCH_HPP
 
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <string>
 
 namespace beast {

include/xrpl/beast/unit_test/reporter.h

@@ -10,8 +10,10 @@
 
 #include <xrpl/beast/unit_test/amount.h>
 #include <xrpl/beast/unit_test/recorder.h>
+
 #include <boost/lexical_cast.hpp>
 #include <boost/optional.hpp>
+
 #include <algorithm>
 #include <chrono>
 #include <functional>

include/xrpl/beast/unit_test/runner.h

@@ -9,7 +9,9 @@
 #define BEAST_UNIT_TEST_RUNNER_H_INCLUDED
 
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <boost/assert.hpp>
+
 #include <mutex>
 #include <ostream>
 #include <string>

include/xrpl/beast/unit_test/suite.h

@@ -9,9 +9,11 @@
 #define BEAST_UNIT_TEST_SUITE_HPP
 
 #include <xrpl/beast/unit_test/runner.h>
+
 #include <boost/filesystem.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/throw_exception.hpp>
+
 #include <ostream>
 #include <sstream>
 #include <string>

include/xrpl/beast/unit_test/suite_list.h

@@ -10,7 +10,9 @@
 
 #include <xrpl/beast/unit_test/detail/const_container.h>
 #include <xrpl/beast/unit_test/suite_info.h>
+
 #include <boost/assert.hpp>
+
 #include <set>
 #include <typeindex>
 #include <unordered_set>

include/xrpl/beast/unit_test/thread.h

@@ -9,6 +9,7 @@
 #define BEAST_UNIT_TEST_THREAD_HPP
 
 #include <xrpl/beast/unit_test/suite.h>
+
 #include <functional>
 #include <thread>
 #include <utility>

include/xrpl/beast/utility/Journal.h

@@ -21,6 +21,7 @@
 #define BEAST_UTILITY_JOURNAL_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <sstream>
 
 namespace beast {

include/xrpl/beast/utility/PropertyStream.h

@@ -22,11 +22,9 @@
 
 #include <xrpl/beast/core/List.h>
 
-#include <cstdint>
 #include <mutex>
 #include <sstream>
 #include <string>
-#include <utility>
 
 namespace beast {
 

include/xrpl/beast/utility/rngfill.h

@@ -21,6 +21,7 @@
 #define BEAST_RANDOM_RNGFILL_H_INCLUDED
 
 #include <xrpl/beast/utility/instrumentation.h>
+
 #include <array>
 #include <cstdint>
 #include <cstring>

include/xrpl/beast/utility/temp_dir.h

@@ -21,6 +21,7 @@
 #define BEAST_UTILITY_TEMP_DIR_H_INCLUDED
 
 #include <boost/filesystem.hpp>
+
 #include <string>
 
 namespace beast {

include/xrpl/crypto/csprng.h

@@ -21,8 +21,6 @@
 #define RIPPLE_CRYPTO_RANDOM_H_INCLUDED
 
 #include <mutex>
-#include <string>
-#include <type_traits>
 
 namespace ripple {
 

include/xrpl/hook/Enum.h

@@ -5,28 +5,6 @@
 #include <vector>
 #ifndef HOOKENUM_INCLUDED
 #define HOOKENUM_INCLUDED 1
-
-#ifndef GUARD_CHECKER_BUILD
-#include <ripple/basics/base_uint.h>
-#include <ripple/protocol/Feature.h>
-#include <ripple/protocol/Rules.h>
-#else
-// Override uint256, Feature and Rules for guard checker build
-#define uint256 std::string
-#define featureHooksUpdate1 "1"
-#define fix20250131 "1"
-namespace hook_api {
-struct Rules
-{
-    constexpr bool
-    enabled(const uint256& feature) const
-    {
-        return true;
-    }
-};
-}  // namespace hook_api
-#endif
-
 namespace ripple {
 enum HookSetOperation : int8_t {
     hsoINVALID = -1,
@@ -389,59 +367,110 @@ const uint8_t max_emit = 255;
 const uint8_t max_params = 16;
 const double fee_base_multiplier = 1.1f;
 
+#define I32 0x7FU
+#define I64 0x7EU
+
+#define HOOK_WRAP_PARAMS(...) __VA_ARGS__
+#define HOOK_API_DEFINITION(RETURN_TYPE, FUNCTION_NAME, PARAMS_TUPLE) \
+    {                                                                 \
+        #FUNCTION_NAME,                                               \
+        {                                                             \
+            RETURN_TYPE, HOOK_WRAP_PARAMS PARAMS_TUPLE                \
+        }                                                             \
+    }
+
 using APIWhitelist = std::map<std::string, std::vector<uint8_t>>;
 
 // RH NOTE: Find descriptions of api functions in ./impl/applyHook.cpp and
 // hookapi.h (include for hooks) this is a map of the api name to its return
 // code (vec[0] and its parameters vec[>0]) as wasm type codes
-inline APIWhitelist
-getImportWhitelist(Rules const& rules)
-{
-    APIWhitelist whitelist;
+static const APIWhitelist import_whitelist{
+    // clang-format off
+    HOOK_API_DEFINITION(I32, _g, (I32, I32)),
+    HOOK_API_DEFINITION(I64, accept, (I32, I32, I64)),
+    HOOK_API_DEFINITION(I64, rollback, (I32, I32, I64)),
+    HOOK_API_DEFINITION(I64, util_raddr, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, util_accid, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, util_verify, (I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, util_sha512h, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, util_keylet, (I32, I32, I32, I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, sto_validate, (I32, I32)),
+    HOOK_API_DEFINITION(I64, sto_subfield, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, sto_subarray, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, sto_emplace, (I32, I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, sto_erase, (I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, etxn_burden, ()),
+    HOOK_API_DEFINITION(I64, etxn_details, (I32, I32)),
+    HOOK_API_DEFINITION(I64, etxn_fee_base, (I32, I32)),
+    HOOK_API_DEFINITION(I64, etxn_reserve, (I32)),
+    HOOK_API_DEFINITION(I64, etxn_generation, ()),
+    HOOK_API_DEFINITION(I64, etxn_nonce, (I32, I32)),
+    HOOK_API_DEFINITION(I64, emit, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, float_set, (I32, I64)),
+    HOOK_API_DEFINITION(I64, float_multiply, (I64, I64)),
+    HOOK_API_DEFINITION(I64, float_mulratio, (I64, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, float_negate, (I64)),
+    HOOK_API_DEFINITION(I64, float_compare, (I64, I64, I32)),
+    HOOK_API_DEFINITION(I64, float_sum, (I64, I64)),
+    HOOK_API_DEFINITION(I64, float_sto, (I32, I32, I32, I32, I32, I32, I64, I32)),
+    HOOK_API_DEFINITION(I64, float_sto_set, (I32, I32)),
+    HOOK_API_DEFINITION(I64, float_invert, (I64)),
+    HOOK_API_DEFINITION(I64, float_divide, (I64, I64)),
+    HOOK_API_DEFINITION(I64, float_one, ()),
+    HOOK_API_DEFINITION(I64, float_mantissa, (I64)),
+    HOOK_API_DEFINITION(I64, float_sign, (I64)),
+    HOOK_API_DEFINITION(I64, float_int, (I64, I32, I32)),
+    HOOK_API_DEFINITION(I64, float_log, (I64)),
+    HOOK_API_DEFINITION(I64, float_root, (I64, I32)),
+    HOOK_API_DEFINITION(I64, fee_base, ()),
+    HOOK_API_DEFINITION(I64, ledger_seq, ()),
+    HOOK_API_DEFINITION(I64, ledger_last_time, ()),
+    HOOK_API_DEFINITION(I64, ledger_last_hash, (I32, I32)),
+    HOOK_API_DEFINITION(I64, ledger_nonce, (I32, I32)),
+    HOOK_API_DEFINITION(I64, ledger_keylet, (I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_account, (I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_hash, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_param_set, (I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_param, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_again, ()),
+    HOOK_API_DEFINITION(I64, hook_skip, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, hook_pos, ()),
+    HOOK_API_DEFINITION(I64, slot, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, slot_clear, (I32)),
+    HOOK_API_DEFINITION(I64, slot_count, (I32)),
+    HOOK_API_DEFINITION(I64, slot_set, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, slot_size, (I32)),
+    HOOK_API_DEFINITION(I64, slot_subarray, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, slot_subfield, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, slot_type, (I32, I32)),
+    HOOK_API_DEFINITION(I64, slot_float, (I32)),
+    HOOK_API_DEFINITION(I64, state_set, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, state_foreign_set, (I32, I32, I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, state, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, state_foreign, (I32, I32, I32, I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, trace, (I32, I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, trace_num, (I32, I32, I64)),
+    HOOK_API_DEFINITION(I64, trace_float, (I32, I32, I64)),
+    HOOK_API_DEFINITION(I64, otxn_burden, ()),
+    HOOK_API_DEFINITION(I64, otxn_field, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, otxn_generation, ()),
+    HOOK_API_DEFINITION(I64, otxn_id, (I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, otxn_type, ()),
+    HOOK_API_DEFINITION(I64, otxn_slot, (I32)),
+    HOOK_API_DEFINITION(I64, otxn_param, (I32, I32, I32, I32)),
+    HOOK_API_DEFINITION(I64, meta_slot, (I32)),
+    // clang-format on
+};
 
-#pragma push_macro("HOOK_API_DEFINITION")
-#undef HOOK_API_DEFINITION
-
-#define int64_t 0x7EU
-#define int32_t 0x7FU
-#define uint32_t 0x7FU
-
-#define HOOK_WRAP_PARAMS(...) __VA_ARGS__
-
-#define HOOK_API_DEFINITION(                                \
-    RETURN_TYPE, FUNCTION_NAME, PARAMS_TUPLE, AMENDMENT)    \
-    if (AMENDMENT == uint256{} || rules.enabled(AMENDMENT)) \
-        whitelist[#FUNCTION_NAME] = {                       \
-            RETURN_TYPE, HOOK_WRAP_PARAMS PARAMS_TUPLE};
-
-#include "hook_api.macro"
-
-#undef HOOK_API_DEFINITION
-#undef HOOK_WRAP_PARAMS
-#undef int64_t
-#undef int32_t
-#undef uint32_t
-#pragma pop_macro("HOOK_API_DEFINITION")
-
-    return whitelist;
-}
+// featureHooks1
+static const APIWhitelist import_whitelist_1{
+    // clang-format off
+    HOOK_API_DEFINITION(I64, xpop_slot, (I32, I32)),
+    // clang-format on
+};
 
 #undef HOOK_API_DEFINITION
 #undef I32
 #undef I64
-
-enum GuardRulesVersion : uint64_t {
-    GuardRuleFix20250131 = 0x00000001,
-};
-
-inline uint64_t
-getGuardRulesVersion(Rules const& rules)
-{
-    uint64_t version = 0;
-    if (rules.enabled(fix20250131))
-        version |= GuardRuleFix20250131;
-    return version;
-}
-
 };  // namespace hook_api
 #endif

include/xrpl/hook/Guard.h

@@ -1,4 +1,5 @@
 #include "Enum.h"
+
 #include <functional>
 #include <iostream>
 #include <map>
@@ -634,7 +635,7 @@ check_guard(
             }
             else if (fc_type == 10)  // memory.copy
             {
-                if (rulesVersion & hook_api::GuardRuleFix20250131)
+                if (rulesVersion & 0x02U)
                     GUARD_ERROR("Memory.copy instruction is not allowed.");
 
                 REQUIRE(2);
@@ -642,7 +643,7 @@ check_guard(
             }
             else if (fc_type == 11)  // memory.fill
             {
-                if (rulesVersion & hook_api::GuardRuleFix20250131)
+                if (rulesVersion & 0x02U)
                     GUARD_ERROR("Memory.fill instruction is not allowed.");
 
                 ADVANCE(1);
@@ -826,7 +827,6 @@ validateGuards(
     std::vector<uint8_t> const& wasm,
     GuardLog guardLog,
     std::string guardLogAccStr,
-    hook_api::APIWhitelist const import_whitelist,
     /* RH NOTE:
      * rules version is a bit field, so rule update 1 is 0x01, update 2 is 0x02
      * and update 3 is 0x04 ideally at rule version 3 all bits so far are set
@@ -836,7 +836,7 @@ validateGuards(
      * might have unforeseen consequences, without also rolling back further
      * changes that are fine.
      */
-    uint64_t rulesVersion = 0x00)
+    uint64_t rulesVersion = 0)
 {
     uint64_t byteCount = wasm.size();
 
@@ -1021,24 +1021,31 @@ validateGuards(
                 int type_idx = parseLeb128(wasm, i, &i);
                 CHECK_SHORT_HOOK();
 
-                auto it = import_whitelist.find(import_name);
-                auto it_end = import_whitelist.end();
-                bool found_in_whitelist = (it != it_end);
-
                 if (import_name == "_g")
                 {
                     guard_import_number = func_upto;
                 }
-                if (!found_in_whitelist)
+                else if (
+                    hook_api::import_whitelist.find(import_name) ==
+                    hook_api::import_whitelist.end())
                 {
-                    GUARDLOG(hook::log::IMPORT_ILLEGAL)
-                        << "Malformed transaction. "
-                        << "Hook attempted to import a function that does "
-                           "not "
-                        << "appear in the hook_api function set: `"
-                        << import_name << "`"
-                        << "\n";
-                    return {};
+                    if (rulesVersion > 0 &&
+                        hook_api::import_whitelist_1.find(import_name) !=
+                            hook_api::import_whitelist_1.end())
+                    {
+                        // PASS, this is a version 1 api
+                    }
+                    else
+                    {
+                        GUARDLOG(hook::log::IMPORT_ILLEGAL)
+                            << "Malformed transaction. "
+                            << "Hook attempted to import a function that does "
+                               "not "
+                            << "appear in the hook_api function set: `"
+                            << import_name << "`"
+                            << "\n";
+                        return {};
+                    }
                 }
 
                 // add to import map
@@ -1253,7 +1260,11 @@ validateGuards(
                     for (auto const& [import_idx, api_name] : usage->second)
                     {
                         auto const& api_signature =
-                            import_whitelist.find(api_name)->second;
+                            hook_api::import_whitelist.find(api_name) !=
+                                hook_api::import_whitelist.end()
+                            ? hook_api::import_whitelist.find(api_name)->second
+                            : hook_api::import_whitelist_1.find(api_name)
+                                  ->second;
 
                         if (!first_signature)
                         {

include/xrpl/hook/HookAPI.h

@@ -1,346 +0,0 @@
-#ifndef HOOK_API_INCLUDED
-#define HOOK_API_INCLUDED 1
-
-#include <xrpld/app/misc/Transaction.h>
-#include <xrpl/hook/Enum.h>
-
-namespace hook {
-using namespace ripple;
-using HookReturnCode = hook_api::hook_return_code;
-
-using Bytes = std::vector<std::uint8_t>;
-
-struct HookContext;  // defined in applyHook.h
-
-class HookAPI
-{
-public:
-    explicit HookAPI(HookContext& ctx) : hookCtx(ctx)
-    {
-    }
-
-    /// control APIs
-    // _g
-    // accept
-    // rollback
-
-    /// util APIs
-    Expected<std::string, HookReturnCode>
-    util_raddr(Bytes const& accountID) const;
-
-    Expected<Bytes, HookReturnCode>
-    util_accid(std::string raddress) const;
-
-    Expected<bool, HookReturnCode>
-    util_verify(Slice const& data, Slice const& sig, Slice const& key) const;
-
-    uint256
-    util_sha512h(Slice const& data) const;
-
-    // util_keylet()
-
-    /// sto APIs
-    Expected<bool, HookReturnCode>
-    sto_validate(Bytes const& data) const;
-
-    Expected<std::pair<uint32_t, uint32_t>, HookReturnCode>
-    sto_subfield(Bytes const& data, uint32_t field_id) const;
-
-    Expected<std::pair<uint32_t, uint32_t>, HookReturnCode>
-    sto_subarray(Bytes const& data, uint32_t index_id) const;
-
-    Expected<Bytes, HookReturnCode>
-    sto_emplace(
-        Bytes const& source_object,
-        std::optional<Bytes> const& field_object,
-        uint32_t field_id) const;
-
-    // sto_erase(): same as sto_emplace with field_object = nullopt
-
-    /// etxn APIs
-    Expected<std::shared_ptr<Transaction>, HookReturnCode>
-    emit(Slice const& txBlob) const;
-
-    Expected<uint64_t, HookReturnCode>
-    etxn_burden() const;
-
-    Expected<uint64_t, HookReturnCode>
-    etxn_fee_base(Slice const& txBlob) const;
-
-    Expected<uint64_t, HookReturnCode>
-    etxn_details(uint8_t* out_ptr) const;
-
-    Expected<uint64_t, HookReturnCode>
-    etxn_reserve(uint64_t count) const;
-
-    uint32_t
-    etxn_generation() const;
-
-    Expected<uint256, HookReturnCode>
-    etxn_nonce() const;
-
-    /// float APIs
-    Expected<uint64_t, HookReturnCode>
-    float_set(int32_t exponent, int64_t mantissa) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_multiply(uint64_t float1, uint64_t float2) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_mulratio(
-        uint64_t float1,
-        uint32_t round_up,
-        uint32_t numerator,
-        uint32_t denominator) const;
-
-    uint64_t
-    float_negate(uint64_t float1) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_compare(uint64_t float1, uint64_t float2, uint32_t mode) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_sum(uint64_t float1, uint64_t float2) const;
-
-    Expected<Bytes, HookReturnCode>
-    float_sto(
-        std::optional<Currency> currency,
-        std::optional<AccountID> issuer,
-        uint64_t float1,
-        uint32_t field_code,
-        uint32_t write_len) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_sto_set(Bytes const& data) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_invert(uint64_t float1) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_divide(uint64_t float1, uint64_t float2) const;
-
-    uint64_t
-    float_one() const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_mantissa(uint64_t float1) const;
-
-    uint64_t
-    float_sign(uint64_t float1) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_int(uint64_t float1, uint32_t decimal_places, uint32_t absolute)
-        const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_log(uint64_t float1) const;
-
-    Expected<uint64_t, HookReturnCode>
-    float_root(uint64_t float1, uint32_t n) const;
-
-    /// otxn APIs
-    uint64_t
-    otxn_burden() const;
-
-    uint32_t
-    otxn_generation() const;
-
-    Expected<const STBase*, HookReturnCode>
-    otxn_field(uint32_t field_id) const;
-
-    Expected<uint256, HookReturnCode>
-    otxn_id(uint32_t flags) const;
-
-    TxType
-    otxn_type() const;
-
-    Expected<uint32_t, HookReturnCode>
-    otxn_slot(uint32_t slot_into) const;
-
-    Expected<Blob, HookReturnCode>
-    otxn_param(Bytes const& param_name) const;
-
-    /// hook APIs
-    AccountID
-    hook_account() const;
-
-    Expected<ripple::uint256, HookReturnCode>
-    hook_hash(int32_t hook_no) const;
-
-    Expected<int64_t, HookReturnCode>
-    hook_again() const;
-
-    Expected<Blob, HookReturnCode>
-    hook_param(Bytes const& paramName) const;
-
-    Expected<uint64_t, HookReturnCode>
-    hook_param_set(
-        uint256 const& hash,
-        Bytes const& paramName,
-        Bytes const& paramValue) const;
-
-    Expected<uint64_t, HookReturnCode>
-    hook_skip(uint256 const& hash, uint32_t flags) const;
-
-    uint8_t
-    hook_pos() const;
-
-    /// ledger APIs
-    uint64_t
-    fee_base() const;
-
-    uint32_t
-    ledger_seq() const;
-
-    uint256
-    ledger_last_hash() const;
-
-    uint64_t
-    ledger_last_time() const;
-
-    Expected<uint256, HookReturnCode>
-    ledger_nonce() const;
-
-    Expected<Keylet, HookReturnCode>
-    ledger_keylet(Keylet const& klLo, Keylet const& klHi) const;
-
-    /// state APIs
-
-    // state(): same as state_foreign with ns = 0 and account = hook_account()
-
-    Expected<Bytes, HookReturnCode>
-    state_foreign(
-        uint256 const& key,
-        uint256 const& ns,
-        AccountID const& account) const;
-
-    // state_set(): same as state_foreign_set with ns = 0 and account =
-
-    Expected<uint64_t, HookReturnCode>
-    state_foreign_set(
-        uint256 const& key,
-        uint256 const& ns,
-        AccountID const& account,
-        Bytes& data) const;
-
-    /// slot APIs
-    Expected<const STBase*, HookReturnCode>
-    slot(uint32_t slot_no) const;
-
-    Expected<uint64_t, HookReturnCode>
-    slot_clear(uint32_t slot_no) const;
-
-    Expected<uint64_t, HookReturnCode>
-    slot_count(uint32_t slot_no) const;
-
-    Expected<uint32_t, HookReturnCode>
-    slot_set(Bytes const& data, uint32_t slot_no) const;
-
-    Expected<uint64_t, HookReturnCode>
-    slot_size(uint32_t slot_no) const;
-
-    Expected<uint32_t, HookReturnCode>
-    slot_subarray(uint32_t parent_slot, uint32_t array_id, uint32_t new_slot)
-        const;
-
-    Expected<uint32_t, HookReturnCode>
-    slot_subfield(uint32_t parent_slot, uint32_t field_id, uint32_t new_slot)
-        const;
-
-    Expected<std::variant<STBase, STAmount>, HookReturnCode>
-    slot_type(uint32_t slot_no, uint32_t flags) const;
-
-    Expected<uint64_t, HookReturnCode>
-    slot_float(uint32_t slot_no) const;
-
-    /// trace APIs
-    // trace
-    // trace_num
-    // trace_float
-
-    Expected<uint32_t, HookReturnCode>
-    meta_slot(uint32_t slot_into) const;
-
-    Expected<std::pair<uint32_t, uint32_t>, HookReturnCode>
-    xpop_slot(uint32_t slot_into_tx, uint32_t slot_into_meta) const;
-
-private:
-    HookContext& hookCtx;
-
-    inline int32_t
-    no_free_slots() const;
-
-    inline std::optional<int32_t>
-    get_free_slot() const;
-
-    inline Expected<uint64_t, HookReturnCode>
-    float_multiply_internal_parts(
-        uint64_t man1,
-        int32_t exp1,
-        bool neg1,
-        uint64_t man2,
-        int32_t exp2,
-        bool neg2) const;
-
-    inline Expected<uint64_t, HookReturnCode>
-    mulratio_internal(
-        int64_t& man1,
-        int32_t& exp1,
-        bool round_up,
-        uint32_t numerator,
-        uint32_t denominator) const;
-
-    inline Expected<uint64_t, HookReturnCode>
-    float_divide_internal(uint64_t float1, uint64_t float2) const;
-
-    inline Expected<uint64_t, HookReturnCode>
-    double_to_xfl(double x) const;
-
-    std::optional<ripple::Keylet>
-    unserialize_keylet(Bytes const& data) const;
-
-    // update the state cache
-    inline std::optional<
-        std::reference_wrapper<std::pair<bool, ripple::Blob> const>>
-    lookup_state_cache(
-        AccountID const& acc,
-        uint256 const& ns,
-        uint256 const& key) const;
-
-    // check the state cache
-    inline Expected<uint64_t, HookReturnCode>
-    set_state_cache(
-        AccountID const& acc,
-        uint256 const& ns,
-        uint256 const& key,
-        Bytes const& data,
-        bool modified) const;
-
-    // these are only used by get_stobject_length below
-    enum parse_error {
-        pe_unexpected_end = -1,
-        pe_unknown_type_early = -2,  // detected early
-        pe_unknown_type_late = -3,   // end of function
-        pe_excessive_nesting = -4,
-        pe_excessive_size = -5
-    };
-
-    inline Expected<
-        int32_t,
-        parse_error>
-    get_stobject_length(
-        unsigned char* start,     // in - begin iterator
-        unsigned char* maxptr,    // in - end iterator
-        int& type,                // out - populated by serialized type code
-        int& field,               // out - populated by serialized field code
-        int& payload_start,       // out - the start of actual payload data for
-                                  // this type
-        int& payload_length,      // out - the length of actual payload data for
-                                  // this type
-        int recursion_depth = 0)  // used internally
-        const;
-};
-
-}  // namespace hook
-
-#endif  // HOOK_API_INCLUDED

include/xrpl/hook/Macro.h

@@ -25,8 +25,7 @@
     _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, N, ...) \
     N
 #define VA_NARGS(__drop, ...) \
-    VA_NARGS_IMPL(            \
-        __VA_OPT__(__VA_ARGS__, ) 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+    VA_NARGS_IMPL(__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1)
 #define FIRST(a, b) a
 #define SECOND(a, b) b
 #define STRIP_TYPES(...) FOR_VARS(SECOND, 0, __VA_ARGS__)
@@ -89,18 +88,30 @@
 
 #define WASM_VAL_TYPE(T, b) CAT2(TYP_, T)
 
-#define DECLARE_HOOK_FUNCTION(R, F, ...)                         \
-    R F(hook::HookContext& hookCtx,                              \
-        WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__( \
-            COMMA __VA_ARGS__));                                 \
-    extern WasmEdge_Result WasmFunction##F(                      \
-        void* data_ptr,                                          \
-        const WasmEdge_CallingFrameContext* frameCtx,            \
-        const WasmEdge_Value* in,                                \
-        WasmEdge_Value* out);                                    \
-    extern WasmEdge_ValType WasmFunctionParams##F[];             \
-    extern WasmEdge_ValType WasmFunctionResult##F[];             \
-    extern WasmEdge_FunctionTypeContext* WasmFunctionType##F;    \
+#define DECLARE_HOOK_FUNCTION(R, F, ...)                      \
+    R F(hook::HookContext& hookCtx,                           \
+        WasmEdge_CallingFrameContext const& frameCtx,         \
+        __VA_ARGS__);                                         \
+    extern WasmEdge_Result WasmFunction##F(                   \
+        void* data_ptr,                                       \
+        const WasmEdge_CallingFrameContext* frameCtx,         \
+        const WasmEdge_Value* in,                             \
+        WasmEdge_Value* out);                                 \
+    extern WasmEdge_ValType WasmFunctionParams##F[];          \
+    extern WasmEdge_ValType WasmFunctionResult##F[];          \
+    extern WasmEdge_FunctionTypeContext* WasmFunctionType##F; \
+    extern WasmEdge_String WasmFunctionName##F;
+
+#define DECLARE_HOOK_FUNCNARG(R, F)                           \
+    R F(hook::HookContext& hookCtx,                           \
+        WasmEdge_CallingFrameContext const& frameCtx);        \
+    extern WasmEdge_Result WasmFunction##F(                   \
+        void* data_ptr,                                       \
+        const WasmEdge_CallingFrameContext* frameCtx,         \
+        const WasmEdge_Value* in,                             \
+        WasmEdge_Value* out);                                 \
+    extern WasmEdge_ValType WasmFunctionResult##F[];          \
+    extern WasmEdge_FunctionTypeContext* WasmFunctionType##F; \
     extern WasmEdge_String WasmFunctionName##F;
 
 #define DEFINE_HOOK_FUNCTION(R, F, ...)                             \
@@ -110,35 +121,61 @@
         const WasmEdge_Value* in,                                   \
         WasmEdge_Value* out)                                        \
     {                                                               \
-        __VA_OPT__(int _stack = 0;)                                 \
-        __VA_OPT__(FOR_VARS(VAR_ASSIGN, 2, __VA_ARGS__);)           \
+        int _stack = 0;                                             \
+        FOR_VARS(VAR_ASSIGN, 2, __VA_ARGS__);                       \
         hook::HookContext* hookCtx =                                \
             reinterpret_cast<hook::HookContext*>(data_ptr);         \
         R return_code = hook_api::F(                                \
             *hookCtx,                                               \
-            *const_cast<WasmEdge_CallingFrameContext*>(frameCtx)    \
-                __VA_OPT__(COMMA STRIP_TYPES(__VA_ARGS__)));        \
+            *const_cast<WasmEdge_CallingFrameContext*>(frameCtx),   \
+            STRIP_TYPES(__VA_ARGS__));                              \
         if (return_code == RC_ROLLBACK || return_code == RC_ACCEPT) \
             return WasmEdge_Result_Terminate;                       \
         out[0] = RET_ASSIGN(R, return_code);                        \
         return WasmEdge_Result_Success;                             \
     };                                                              \
     WasmEdge_ValType hook_api::WasmFunctionParams##F[] = {          \
-        __VA_OPT__(FOR_VARS(WASM_VAL_TYPE, 0, __VA_ARGS__))};       \
+        FOR_VARS(WASM_VAL_TYPE, 0, __VA_ARGS__)};                   \
     WasmEdge_ValType hook_api::WasmFunctionResult##F[1] = {         \
         WASM_VAL_TYPE(R, dummy)};                                   \
     WasmEdge_FunctionTypeContext* hook_api::WasmFunctionType##F =   \
         WasmEdge_FunctionTypeCreate(                                \
             WasmFunctionParams##F,                                  \
-            VA_NARGS(NULL __VA_OPT__(, __VA_ARGS__)),               \
+            VA_NARGS(NULL, __VA_ARGS__),                            \
             WasmFunctionResult##F,                                  \
             1);                                                     \
     WasmEdge_String hook_api::WasmFunctionName##F =                 \
         WasmEdge_StringCreateByCString(#F);                         \
     R hook_api::F(                                                  \
         hook::HookContext& hookCtx,                                 \
-        WasmEdge_CallingFrameContext const& frameCtx __VA_OPT__(    \
-            COMMA __VA_ARGS__))
+        WasmEdge_CallingFrameContext const& frameCtx,               \
+        __VA_ARGS__)
+
+#define DEFINE_HOOK_FUNCNARG(R, F)                                           \
+    WasmEdge_Result hook_api::WasmFunction##F(                               \
+        void* data_ptr,                                                      \
+        const WasmEdge_CallingFrameContext* frameCtx,                        \
+        const WasmEdge_Value* in,                                            \
+        WasmEdge_Value* out)                                                 \
+    {                                                                        \
+        hook::HookContext* hookCtx =                                         \
+            reinterpret_cast<hook::HookContext*>(data_ptr);                  \
+        R return_code = hook_api::F(                                         \
+            *hookCtx, *const_cast<WasmEdge_CallingFrameContext*>(frameCtx)); \
+        if (return_code == RC_ROLLBACK || return_code == RC_ACCEPT)          \
+            return WasmEdge_Result_Terminate;                                \
+        out[0] = CAT2(RET_, R(return_code));                                 \
+        return WasmEdge_Result_Success;                                      \
+    };                                                                       \
+    WasmEdge_ValType hook_api::WasmFunctionResult##F[1] = {                  \
+        WASM_VAL_TYPE(R, dummy)};                                            \
+    WasmEdge_FunctionTypeContext* hook_api::WasmFunctionType##F =            \
+        WasmEdge_FunctionTypeCreate({}, 0, WasmFunctionResult##F, 1);        \
+    WasmEdge_String hook_api::WasmFunctionName##F =                          \
+        WasmEdge_StringCreateByCString(#F);                                  \
+    R hook_api::F(                                                           \
+        hook::HookContext& hookCtx,                                          \
+        WasmEdge_CallingFrameContext const& frameCtx)
 
 #define HOOK_SETUP()                                                 \
     try                                                              \
@@ -153,7 +190,6 @@
         [[maybe_unused]] const uint64_t memory_length =              \
             WasmEdge_MemoryInstanceGetPageSize(memoryCtx) *          \
             WasmEdge_kPageSize;                                      \
-        [[maybe_unused]] auto& api = hookCtx.api();                  \
         if (!memoryCtx || !memory || !memory_length)                 \
             return INTERNAL_ERROR;
 

include/xrpl/hook/guard_checker.cpp

@@ -1,17 +1,17 @@
-#define GUARD_CHECKER_BUILD
-#include "Enum.h"
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+
 #include "Guard.h"
 #include <fcntl.h>
-#include <iostream>
-#include <optional>
-#include <ostream>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
-#include <string_view>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
 #include <unistd.h>
+
+#include <iostream>
+#include <optional>
+#include <ostream>
+#include <string_view>
 #include <vector>
 
 int
@@ -81,15 +81,7 @@ main(int argc, char** argv)
 
     close(fd);
 
-    // Dummy rules for guard checker build
-    hook_api::Rules rules;
-
-    auto result = validateGuards(
-        hook,
-        std::cout,
-        "",
-        hook_api::getImportWhitelist(rules),
-        hook_api::getGuardRulesVersion(rules));
+    auto result = validateGuards(hook, std::cout, "", 3);
 
     if (!result)
     {

include/xrpl/hook/hook_api.macro

@@ -1,369 +0,0 @@
-// int32_t _g(uint32_t guard_id, uint32_t maxiter);
-HOOK_API_DEFINITION(
-  int32_t, _g, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t accept(uint32_t read_ptr, uint32_t read_len, int64_t error_code);
-HOOK_API_DEFINITION(
-  int64_t, accept, (uint32_t, uint32_t, int64_t),
-  uint256{})
-
-// int64_t rollback(uint32_t read_ptr, uint32_t read_len, int64_t error_code);
-HOOK_API_DEFINITION(
-  int64_t, rollback, (uint32_t, uint32_t, int64_t),
-  uint256{})
-
-// int64_t util_raddr(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, util_raddr, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t util_accid(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, util_accid, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t util_verify(uint32_t dread_ptr, uint32_t dread_len, uint32_t sread_ptr, uint32_t sread_len, uint32_t kread_ptr, uint32_t kread_len);
-HOOK_API_DEFINITION(
-  int64_t, util_verify, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t util_sha512h(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, util_sha512h, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t util_keylet(uint32_t write_ptr, uint32_t write_len, uint32_t keylet_type, uint32_t a, uint32_t b, uint32_t c, uint32_t d, uint32_t e, uint32_t f);
-HOOK_API_DEFINITION(
-  int64_t, util_keylet, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t sto_validate(uint32_t tread_ptr, uint32_t tread_len);
-HOOK_API_DEFINITION(
-  int64_t, sto_validate, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t sto_subfield(uint32_t read_ptr, uint32_t read_len, uint32_t field_id);
-HOOK_API_DEFINITION(
-  int64_t, sto_subfield, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t sto_subarray(uint32_t read_ptr, uint32_t read_len, uint32_t array_id);
-HOOK_API_DEFINITION(
-  int64_t, sto_subarray, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t sto_emplace(uint32_t write_ptr, uint32_t write_len, uint32_t sread_ptr, uint32_t sread_len, uint32_t fread_ptr, uint32_t fread_len, uint32_t field_id);
-HOOK_API_DEFINITION(
-  int64_t, sto_emplace, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t sto_erase(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len, uint32_t field_id);
-HOOK_API_DEFINITION(
-  int64_t, sto_erase, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t etxn_burden();
-HOOK_API_DEFINITION(
-  int64_t, etxn_burden, (),
-  uint256{})
-
-// int64_t etxn_details(uint32_t write_ptr, uint32_t write_len);
-HOOK_API_DEFINITION(
-  int64_t, etxn_details, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t etxn_fee_base(uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, etxn_fee_base, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t etxn_reserve(uint32_t count);
-HOOK_API_DEFINITION(
-  int64_t, etxn_reserve, (uint32_t),
-  uint256{})
-
-// int64_t etxn_generation();
-HOOK_API_DEFINITION(
-  int64_t, etxn_generation, (),
-  uint256{})
-
-// int64_t etxn_nonce(uint32_t write_ptr, uint32_t write_len);
-HOOK_API_DEFINITION(
-  int64_t, etxn_nonce, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t emit(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, emit, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t float_set(int32_t exponent, int64_t mantissa);
-HOOK_API_DEFINITION(
-  int64_t, float_set, (int32_t, int64_t),
-  uint256{})
-
-// int64_t float_multiply(int64_t float1, int64_t float2);
-HOOK_API_DEFINITION(
-  int64_t, float_multiply, (int64_t, int64_t),
-  uint256{})
-
-// int64_t float_mulratio(int64_t float1, uint32_t round_up, uint32_t numerator, uint32_t denominator);
-HOOK_API_DEFINITION(
-  int64_t, float_mulratio, (int64_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t float_negate(int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, float_negate, (int64_t),
-  uint256{})
-
-// int64_t float_compare(int64_t float1, int64_t float2, uint32_t mode);
-HOOK_API_DEFINITION(
-  int64_t, float_compare, (int64_t, int64_t, uint32_t),
-  uint256{})
-
-// int64_t float_sum(int64_t float1, int64_t float2);
-HOOK_API_DEFINITION(
-  int64_t, float_sum, (int64_t, int64_t),
-  uint256{})
-
-// int64_t float_sto(uint32_t write_ptr, uint32_t write_len, uint32_t cread_ptr, uint32_t cread_len, uint32_t iread_ptr, uint32_t iread_len, int64_t float1, uint32_t field_code);
-HOOK_API_DEFINITION(
-  int64_t, float_sto, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, int64_t, uint32_t),
-  uint256{})
-
-// int64_t float_sto_set(uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, float_sto_set, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t float_invert(int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, float_invert, (int64_t),
-  uint256{})
-
-// int64_t float_divide(int64_t float1, int64_t float2);
-HOOK_API_DEFINITION(
-  int64_t, float_divide, (int64_t, int64_t),
-  uint256{})
-
-// int64_t float_one();
-HOOK_API_DEFINITION(
-  int64_t, float_one, (),
-  uint256{})
-
-// int64_t float_mantissa(int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, float_mantissa, (int64_t),
-  uint256{})
-
-// int64_t float_sign(int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, float_sign, (int64_t),
-  uint256{})
-
-// int64_t float_int(int64_t float1, uint32_t decimal_places, uint32_t abs);
-HOOK_API_DEFINITION(
-  int64_t, float_int, (int64_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t float_log(int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, float_log, (int64_t),
-  uint256{})
-
-// int64_t float_root(int64_t float1, uint32_t n);
-HOOK_API_DEFINITION(
-  int64_t, float_root, (int64_t, uint32_t),
-  uint256{})
-
-// int64_t fee_base();
-HOOK_API_DEFINITION(
-  int64_t, fee_base, (),
-  uint256{})
-
-// int64_t ledger_seq();
-HOOK_API_DEFINITION(
-  int64_t, ledger_seq, (),
-  uint256{})
-
-// int64_t ledger_last_time();
-HOOK_API_DEFINITION(
-  int64_t, ledger_last_time, (),
-  uint256{})
-
-// int64_t ledger_last_hash(uint32_t write_ptr, uint32_t write_len);
-HOOK_API_DEFINITION(
-  int64_t, ledger_last_hash, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t ledger_nonce(uint32_t write_ptr, uint32_t write_len);
-HOOK_API_DEFINITION(
-  int64_t, ledger_nonce, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t ledger_keylet(uint32_t write_ptr, uint32_t write_len, uint32_t lread_ptr, uint32_t lread_len, uint32_t hread_ptr, uint32_t hread_len);
-HOOK_API_DEFINITION(
-  int64_t, ledger_keylet, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t hook_account(uint32_t write_ptr, uint32_t write_len);
-HOOK_API_DEFINITION(
-  int64_t, hook_account, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t hook_hash(uint32_t write_ptr, uint32_t write_len, int32_t hook_no);
-HOOK_API_DEFINITION(
-  int64_t, hook_hash, (uint32_t, uint32_t, int32_t),
-  uint256{})
-
-// int64_t hook_param_set(uint32_t read_ptr, uint32_t read_len, uint32_t kread_ptr, uint32_t kread_len, uint32_t hread_ptr, uint32_t hread_len);
-HOOK_API_DEFINITION(
-  int64_t, hook_param_set, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t hook_param(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, hook_param, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t hook_again();
-HOOK_API_DEFINITION(
-  int64_t, hook_again, (),
-  uint256{})
-
-// int64_t hook_skip(uint32_t read_ptr, uint32_t read_len, uint32_t flags);
-HOOK_API_DEFINITION(
-  int64_t, hook_skip, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t hook_pos();
-HOOK_API_DEFINITION(
-  int64_t, hook_pos, (),
-  uint256{})
-
-// int64_t slot(uint32_t write_ptr, uint32_t write_len, uint32_t slot);
-HOOK_API_DEFINITION(
-  int64_t, slot, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t slot_clear(uint32_t slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_clear, (uint32_t),
-  uint256{})
-
-// int64_t slot_count(uint32_t slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_count, (uint32_t),
-  uint256{})
-
-// int64_t slot_set(uint32_t read_ptr, uint32_t read_len, uint32_t slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_set, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t slot_size(uint32_t slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_size, (uint32_t),
-  uint256{})
-
-// int64_t slot_subarray(uint32_t parent_slot, uint32_t array_id, uint32_t new_slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_subarray, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t slot_subfield(uint32_t parent_slot, uint32_t field_id, uint32_t new_slot);
-HOOK_API_DEFINITION(
-  int64_t, slot_subfield, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t slot_type(uint32_t slot_no, uint32_t flags);
-HOOK_API_DEFINITION(
-  int64_t, slot_type, (uint32_t, uint32_t),
-  uint256{})
-
-// int64_t slot_float(uint32_t slot_no);
-HOOK_API_DEFINITION(
-  int64_t, slot_float, (uint32_t),
-  uint256{})
-
-// int64_t state_set(uint32_t read_ptr, uint32_t read_len, uint32_t kread_ptr, uint32_t kread_len);
-HOOK_API_DEFINITION(
-  int64_t, state_set, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t state_foreign_set(uint32_t read_ptr, uint32_t read_len, uint32_t kread_ptr, uint32_t kread_len, uint32_t nread_ptr, uint32_t nread_len, uint32_t aread_ptr, uint32_t aread_len);
-HOOK_API_DEFINITION(
-  int64_t, state_foreign_set, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t state(uint32_t write_ptr, uint32_t write_len, uint32_t kread_ptr, uint32_t kread_len);
-HOOK_API_DEFINITION(
-  int64_t, state, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t state_foreign(uint32_t write_ptr, uint32_t write_len, uint32_t kread_ptr, uint32_t kread_len, uint32_t nread_ptr, uint32_t nread_len, uint32_t aread_ptr, uint32_t aread_len);
-HOOK_API_DEFINITION(
-  int64_t, state_foreign, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t trace(uint32_t mread_ptr, uint32_t mread_len, uint32_t dread_ptr, uint32_t dread_len, uint32_t as_hex);
-HOOK_API_DEFINITION(
-  int64_t, trace, (uint32_t, uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t trace_num(uint32_t read_ptr, uint32_t read_len, int64_t number);
-HOOK_API_DEFINITION(
-  int64_t, trace_num, (uint32_t, uint32_t, int64_t),
-  uint256{})
-
-// int64_t trace_float(uint32_t read_ptr, uint32_t read_len, int64_t float1);
-HOOK_API_DEFINITION(
-  int64_t, trace_float, (uint32_t, uint32_t, int64_t),
-  uint256{})
-
-// int64_t otxn_burden();
-HOOK_API_DEFINITION(
-  int64_t, otxn_burden, (),
-  uint256{})
-
-// int64_t otxn_field(uint32_t write_ptr, uint32_t write_len, uint32_t field_id);
-HOOK_API_DEFINITION(
-  int64_t, otxn_field, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t otxn_generation();
-HOOK_API_DEFINITION(
-  int64_t, otxn_generation, (),
-  uint256{})
-
-// int64_t otxn_id(uint32_t write_ptr, uint32_t write_len, uint32_t flags);
-HOOK_API_DEFINITION(
-  int64_t, otxn_id, (uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t otxn_type();
-HOOK_API_DEFINITION(
-  int64_t, otxn_type, (),
-  uint256{})
-
-// int64_t otxn_slot(uint32_t slot_no);
-HOOK_API_DEFINITION(
-  int64_t, otxn_slot, (uint32_t),
-  uint256{})
-
-// int64_t otxn_param(uint32_t write_ptr, uint32_t write_len, uint32_t read_ptr, uint32_t read_len);
-HOOK_API_DEFINITION(
-  int64_t, otxn_param, (uint32_t, uint32_t, uint32_t, uint32_t),
-  uint256{})
-
-// int64_t meta_slot(uint32_t slot_no);
-HOOK_API_DEFINITION(
-  int64_t, meta_slot, (uint32_t),
-  uint256{})
-
-// int64_t xpop_slot(uint32_t slot_no_tx, uint32_t slot_no_meta);
-HOOK_API_DEFINITION(
-  int64_t, xpop_slot, (uint32_t, uint32_t),
-  featureHooksUpdate1)

include/xrpl/json/Object.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_OBJECT_H_INCLUDED
 
 #include <xrpl/json/Writer.h>
+
 #include <memory>
 
 namespace Json {

include/xrpl/json/Output.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_OUTPUT_H_INCLUDED
 
 #include <boost/beast/core/string.hpp>
+
 #include <functional>
 #include <string>
 

include/xrpl/json/Writer.h

@@ -24,6 +24,7 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/json/Output.h>
 #include <xrpl/json/json_value.h>
+
 #include <memory>
 
 namespace Json {

include/xrpl/json/detail/json_assert.h

@@ -20,6 +20,7 @@
 #ifndef RIPPLE_JSON_JSON_ASSERT_H_INCLUDED
 #define RIPPLE_JSON_JSON_ASSERT_H_INCLUDED
 
+#include <xrpl/basics/contract.h>
 #include <xrpl/json/json_errors.h>
 
 #define JSON_ASSERT_MESSAGE(condition, message) \

include/xrpl/json/json_reader.h

@@ -24,7 +24,9 @@
 
 #include <xrpl/json/json_forwards.h>
 #include <xrpl/json/json_value.h>
+
 #include <boost/asio/buffer.hpp>
+
 #include <stack>
 
 namespace Json {

include/xrpl/json/json_value.h

@@ -21,6 +21,7 @@
 #define RIPPLE_JSON_JSON_VALUE_H_INCLUDED
 
 #include <xrpl/json/json_forwards.h>
+
 #include <cstring>
 #include <map>
 #include <string>

include/xrpl/json/json_writer.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/json/json_forwards.h>
 #include <xrpl/json/json_value.h>
+
 #include <ostream>
 #include <vector>
 

include/xrpl/protocol/Book.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/CountedObject.h>
 #include <xrpl/protocol/Issue.h>
+
 #include <boost/utility/base_from_member.hpp>
 
 namespace ripple {

include/xrpl/protocol/Feature.h

@@ -21,7 +21,9 @@
 #define RIPPLE_PROTOCOL_FEATURE_H_INCLUDED
 
 #include <xrpl/basics/base_uint.h>
+
 #include <boost/container/flat_map.hpp>
+
 #include <array>
 #include <bitset>
 #include <map>
@@ -33,35 +35,39 @@
  *
  * Steps required to add new features to the code:
  *
- * 1) In this file, increment `numFeatures` and add a uint256 declaration
- *    for the feature at the bottom
- * 2) Add a uint256 definition for the feature to the corresponding source
- *    file (Feature.cpp). Use `registerFeature` to create the feature with
- *    the feature's name, `Supported::no`, and `VoteBehavior::DefaultNo`. This
- *    should be the only place the feature's name appears in code as a string.
- * 3) Use the uint256 as the parameter to `view.rules.enabled()` to
- *    control flow into new code that this feature limits.
- * 4) If the feature development is COMPLETE, and the feature is ready to be
- *    SUPPORTED, change the `registerFeature` parameter to Supported::yes.
- * 5) When the feature is ready to be ENABLED, change the `registerFeature`
- *    parameter to `VoteBehavior::DefaultYes`.
- * In general, any newly supported amendments (`Supported::yes`) should have
- * a `VoteBehavior::DefaultNo` for at least one full release cycle. High
- * priority bug fixes can be an exception to this rule of thumb.
+ * 1) Add the appropriate XRPL_FEATURE or XRPL_FIX macro definition for the
+ *    feature to features.macro with the feature's name, `Supported::no`, and
+ *    `VoteBehavior::DefaultNo`.
+ *
+ * 2) Use the generated variable name as the parameter to `view.rules.enabled()`
+ *    to control flow into new code that this feature limits. (featureName or
+ *    fixName)
+ *
+ * 3) If the feature development is COMPLETE, and the feature is ready to be
+ *    SUPPORTED, change the macro parameter in features.macro to Supported::yes.
+ *
+ * 4) In general, any newly supported amendments (`Supported::yes`) should have
+ *    a `VoteBehavior::DefaultNo` indefinitely so that external governance can
+ *    make the decision on when to activate it. High priority bug fixes can be
+ *    an exception to this rule. In such cases, ensure the fix has been
+ *    clearly communicated to the community using appropriate channels,
+ *    then change the macro parameter in features.macro to
+ *    `VoteBehavior::DefaultYes`. The communication process is beyond
+ *    the scope of these instructions.
+ *
  *
  * When a feature has been enabled for several years, the conditional code
  * may be removed, and the feature "retired". To retire a feature:
- * 1) Remove the uint256 declaration from this file.
- * 2) MOVE the uint256 definition in Feature.cpp to the "retired features"
- *    section at the end of the file.
- * 3) CHANGE the name of the variable to start with "retired".
- * 4) CHANGE the parameters of the `registerFeature` call to `Supported::yes`
- *    and `VoteBehavior::DefaultNo`.
+ *
+ * 1) MOVE the macro definition in features.macro to the "retired features"
+ *    section at the end of the file, and change the macro to XRPL_RETIRE.
+ *
  * The feature must remain registered and supported indefinitely because it
- * still exists in the ledger, but there is no need to vote for it because
- * there's nothing to vote for. If it is removed completely from the code, any
- * instances running that code will get amendment blocked. Removing the
- * feature from the ledger is beyond the scope of these instructions.
+ * may exist in the Amendments object on ledger. There is no need to vote
+ * for it because there's nothing to vote for. If the feature definition is
+ * removed completely from the code, any instances running that code will get
+ * amendment blocked. Removing the feature from the ledger is beyond the scope
+ * of these instructions.
  *
  */
 
@@ -76,11 +82,32 @@ allAmendments();
 
 namespace detail {
 
+#pragma push_macro("XRPL_FEATURE")
+#undef XRPL_FEATURE
+#pragma push_macro("XRPL_FIX")
+#undef XRPL_FIX
+#pragma push_macro("XRPL_RETIRE")
+#undef XRPL_RETIRE
+
+#define XRPL_FEATURE(name, supported, vote) +1
+#define XRPL_FIX(name, supported, vote) +1
+#define XRPL_RETIRE(name) +1
+
 // This value SHOULD be equal to the number of amendments registered in
 // Feature.cpp. Because it's only used to reserve storage, and determine how
 // large to make the FeatureBitset, it MAY be larger. It MUST NOT be less than
 // the actual number of amendments. A LogicError on startup will verify this.
-static constexpr std::size_t numFeatures = 110;
+static constexpr std::size_t numFeatures =
+    (0 +
+#include <xrpl/protocol/detail/features.macro>
+    );
+
+#undef XRPL_RETIRE
+#pragma pop_macro("XRPL_RETIRE")
+#undef XRPL_FIX
+#pragma pop_macro("XRPL_FIX")
+#undef XRPL_FEATURE
+#pragma pop_macro("XRPL_FEATURE")
 
 /** Amendments that this server supports and the default voting behavior.
    Whether they are enabled depends on the Rules defined in the validated
@@ -320,12 +347,17 @@ foreachFeature(FeatureBitset bs, F&& f)
 #undef XRPL_FEATURE
 #pragma push_macro("XRPL_FIX")
 #undef XRPL_FIX
+#pragma push_macro("XRPL_RETIRE")
+#undef XRPL_RETIRE
 
 #define XRPL_FEATURE(name, supported, vote) extern uint256 const feature##name;
 #define XRPL_FIX(name, supported, vote) extern uint256 const fix##name;
+#define XRPL_RETIRE(name)
 
 #include <xrpl/protocol/detail/features.macro>
 
+#undef XRPL_RETIRE
+#pragma pop_macro("XRPL_RETIRE")
 #undef XRPL_FIX
 #pragma pop_macro("XRPL_FIX")
 #undef XRPL_FEATURE

include/xrpl/protocol/FeeUnits.h

@@ -23,6 +23,7 @@
 #include <xrpl/beast/utility/Zero.h>
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/json/json_value.h>
+
 #include <boost/multiprecision/cpp_int.hpp>
 #include <boost/operators.hpp>
 

include/xrpl/protocol/HashPrefix.h

@@ -21,6 +21,7 @@
 #define RIPPLE_PROTOCOL_HASHPREFIX_H_INCLUDED
 
 #include <xrpl/beast/hash/hash_append.h>
+
 #include <cstdint>
 
 namespace ripple {

include/xrpl/protocol/IOUAmount.h

@@ -23,7 +23,9 @@
 #include <xrpl/basics/LocalValue.h>
 #include <xrpl/basics/Number.h>
 #include <xrpl/beast/utility/Zero.h>
+
 #include <boost/operators.hpp>
+
 #include <cstdint>
 #include <string>
 #include <utility>

include/xrpl/protocol/Import.h

@@ -22,9 +22,11 @@
 
 // #include <ripple/basics/Log.h>
 #include <xrpld/app/misc/Manifest.h>
+
 #include <xrpl/basics/StringUtilities.h>
 #include <xrpl/basics/base64.h>
 #include <xrpl/json/json_reader.h>
+
 #include <charconv>
 
 namespace ripple {

include/xrpl/protocol/KnownFormats.h

@@ -23,7 +23,9 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/beast/type_name.h>
 #include <xrpl/protocol/SOTemplate.h>
+
 #include <boost/container/flat_map.hpp>
+
 #include <algorithm>
 #include <forward_list>
 

include/xrpl/protocol/MultiApiJson.h

@@ -20,10 +20,10 @@
 #ifndef RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED
 #define RIPPLE_JSON_MULTIAPIJSON_H_INCLUDED
 
+#include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/json/json_value.h>
 #include <xrpl/protocol/ApiVersion.h>
 
-#include <xrpl/beast/utility/instrumentation.h>
 #include <array>
 #include <concepts>
 #include <cstdlib>

include/xrpl/protocol/Protocol.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/ByteUtilities.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/basics/partitioned_unordered_map.h>
+
 #include <cstdint>
 
 namespace ripple {

include/xrpl/protocol/Rate.h

@@ -22,7 +22,9 @@
 
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/protocol/STAmount.h>
+
 #include <boost/operators.hpp>
+
 #include <cstdint>
 #include <ostream>
 

include/xrpl/protocol/SOTemplate.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/contract.h>
 #include <xrpl/protocol/SField.h>
+
 #include <functional>
 #include <initializer_list>
 #include <memory>

include/xrpl/protocol/STBase.h

@@ -23,12 +23,14 @@
 #include <xrpl/basics/contract.h>
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/Serializer.h>
+
 #include <memory>
 #include <ostream>
 #include <string>
 #include <type_traits>
 #include <typeinfo>
 #include <utility>
+
 namespace ripple {
 
 /// Note, should be treated as flags that can be | and &

include/xrpl/protocol/STExchange.h

@@ -28,6 +28,7 @@
 #include <xrpl/protocol/STBlob.h>
 #include <xrpl/protocol/STInteger.h>
 #include <xrpl/protocol/STObject.h>
+
 #include <memory>
 #include <optional>
 #include <stdexcept>

include/xrpl/protocol/STLedgerEntry.h

@@ -124,4 +124,4 @@ STLedgerEntry::getType() const
 
 }  // namespace ripple
 
-#endif
+#endif

include/xrpl/protocol/STObject.h

@@ -35,7 +35,9 @@
 #include <xrpl/protocol/STPathSet.h>
 #include <xrpl/protocol/STVector256.h>
 #include <xrpl/protocol/detail/STVar.h>
+
 #include <boost/iterator/transform_iterator.hpp>
+
 #include <optional>
 #include <stdexcept>
 #include <type_traits>

include/xrpl/protocol/STParsedJSON.h

@@ -21,6 +21,7 @@
 #define RIPPLE_PROTOCOL_STPARSEDJSON_H_INCLUDED
 
 #include <xrpl/protocol/STArray.h>
+
 #include <optional>
 
 namespace ripple {

include/xrpl/protocol/STPathSet.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/SField.h>
 #include <xrpl/protocol/STBase.h>
 #include <xrpl/protocol/UintTypes.h>
+
 #include <cstddef>
 #include <optional>
 

include/xrpl/protocol/STTx.h

@@ -28,6 +28,7 @@
 #include <xrpl/protocol/SecretKey.h>
 #include <xrpl/protocol/SeqProxy.h>
 #include <xrpl/protocol/TxFormats.h>
+
 #include <boost/container/flat_set.hpp>
 
 #include <functional>

include/xrpl/protocol/STValidation.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/SecretKey.h>
+
 #include <cstdint>
 #include <functional>
 #include <memory>
@@ -146,8 +147,7 @@ public:
     render() const
     {
         std::stringstream ss;
-        ss << "validation: "
-           << " ledger_hash: " << getLedgerHash()
+        ss << "validation: " << " ledger_hash: " << getLedgerHash()
            << " consensus_hash: " << getConsensusHash()
            << " sign_time: " << to_string(getSignTime())
            << " seen_time: " << to_string(getSeenTime())

include/xrpl/protocol/SecretKey.h

@@ -26,6 +26,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/Seed.h>
 #include <xrpl/protocol/tokens.h>
+
 #include <array>
 #include <cstring>
 #include <string>

include/xrpl/protocol/Seed.h

@@ -23,6 +23,7 @@
 #include <xrpl/basics/Slice.h>
 #include <xrpl/basics/base_uint.h>
 #include <xrpl/protocol/tokens.h>
+
 #include <array>
 #include <optional>
 

include/xrpl/protocol/Serializer.h

@@ -30,6 +30,7 @@
 #include <xrpl/beast/utility/instrumentation.h>
 #include <xrpl/protocol/HashPrefix.h>
 #include <xrpl/protocol/SField.h>
+
 #include <cstdint>
 #include <cstring>
 #include <iomanip>

include/xrpl/protocol/Sign.h

@@ -24,6 +24,7 @@
 #include <xrpl/protocol/PublicKey.h>
 #include <xrpl/protocol/STObject.h>
 #include <xrpl/protocol/SecretKey.h>
+
 #include <utility>
 
 namespace ripple {

include/xrpl/protocol/SystemParameters.h

@@ -22,6 +22,7 @@
 
 #include <xrpl/basics/chrono.h>
 #include <xrpl/protocol/XRPAmount.h>
+
 #include <cstdint>
 #include <string>