add compression (untested) fix some build system issues

2026-06-02 16:26:37 +00:00 · 2025-03-23 14:22:54 +11:00
parent 63077ac753
commit c5d298d6e8
6 changed files with 200 additions and 59 deletions
--- a/Builds/CMake/deps/Boost.cmake
+++ b/Builds/CMake/deps/Boost.cmake
@@ -1,14 +1,16 @@
 #[===================================================================[
   NIH dep: boost
 #]===================================================================]
-
 if((NOT DEFINED BOOST_ROOT) AND(DEFINED ENV{BOOST_ROOT}))
  set(BOOST_ROOT $ENV{BOOST_ROOT})
 endif()
+if((NOT DEFINED BOOST_LIBRARYDIR) AND(DEFINED ENV{BOOST_LIBRARYDIR}))
+  set(BOOST_LIBRARYDIR $ENV{BOOST_LIBRARYDIR})
+endif()
 file(TO_CMAKE_PATH "${BOOST_ROOT}" BOOST_ROOT)
 if(WIN32 OR CYGWIN)
  # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders
-  if(DEFINED BOOST_ROOT)
+  if((NOT DEFINED BOOST_LIBRARYDIR) AND (DEFINED BOOST_ROOT))
    if(IS_DIRECTORY ${BOOST_ROOT}/stage64/lib)
      set(BOOST_LIBRARYDIR ${BOOST_ROOT}/stage64/lib)
    elseif(IS_DIRECTORY ${BOOST_ROOT}/stage/lib)
@@ -55,6 +57,7 @@ find_package(Boost 1.86 REQUIRED
    program_options
    regex
    system
+    iostreams
    thread)

 add_library(ripple_boost INTERFACE)
@@ -74,6 +77,7 @@ target_link_libraries(ripple_boost
    Boost::coroutine
    Boost::date_time
    Boost::filesystem
+    Boost::iostreams
    Boost::program_options
    Boost::regex
    Boost::system
--- a/Builds/CMake/deps/FindBoost.cmake
+++ b/Builds/CMake/deps/FindBoost.cmake
@@ -248,6 +248,7 @@ include(FindPackageHandleStandardArgs)
 # Save project's policies
 cmake_policy(PUSH)
 cmake_policy(SET CMP0057 NEW) # if IN_LIST
+cmake_policy(SET CMP0144 NEW)

 #-------------------------------------------------------------------------------
 # Before we go searching, check whether a boost cmake package is available, unless
@@ -969,7 +970,24 @@ function(_Boost_COMPONENT_DEPENDENCIES component _ret)
      set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono date_time atomic)
      set(_Boost_WSERIALIZATION_DEPENDENCIES serialization)
    endif()
-    if(NOT Boost_VERSION_STRING VERSION_LESS 1.77.0)
+    
+    # Special handling for Boost 1.86.0 and higher
+    if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0)
+      # Explicitly set these for Boost 1.86
+      set(_Boost_IOSTREAMS_DEPENDENCIES "")  # No dependencies for iostreams in 1.86
+      
+      # Debug output to help diagnose the issue
+      if(Boost_DEBUG)
+        message(STATUS "Using special dependency settings for Boost 1.86.0+")
+        message(STATUS "Component: ${component}, uppercomponent: ${uppercomponent}")
+        message(STATUS "Boost_VERSION_STRING: ${Boost_VERSION_STRING}")
+        message(STATUS "BOOST_ROOT: $ENV{BOOST_ROOT}")
+        message(STATUS "BOOST_LIBRARYDIR: $ENV{BOOST_LIBRARYDIR}")
+      endif()
+    endif()
+    
+    # Only show warning for versions beyond what we've defined
+    if(NOT Boost_VERSION_STRING VERSION_LESS 1.87.0)
      message(WARNING "New Boost version may have incorrect or missing dependencies and imported targets")
    endif()
  endif()
@@ -1879,6 +1897,18 @@ foreach(COMPONENT ${Boost_FIND_COMPONENTS})
    list(INSERT _boost_LIBRARY_SEARCH_DIRS_RELEASE 0 ${Boost_LIBRARY_DIR_DEBUG})
  endif()

+  if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0)
+    if(BOOST_LIBRARYDIR AND EXISTS "${BOOST_LIBRARYDIR}")
+      # Clear existing search paths and use only BOOST_LIBRARYDIR 
+      set(_boost_LIBRARY_SEARCH_DIRS_RELEASE "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH)
+      set(_boost_LIBRARY_SEARCH_DIRS_DEBUG "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH)
+        
+      if(Boost_DEBUG)
+        message(STATUS "Boost 1.86: Setting library search dirs to BOOST_LIBRARYDIR: ${BOOST_LIBRARYDIR}")
+      endif()
+    endif()
+  endif()
+
  # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing.
  string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS_RELEASE}")

--- a/Builds/CMake/deps/gRPC.cmake
+++ b/Builds/CMake/deps/gRPC.cmake
@@ -74,7 +74,11 @@ else ()
      if (NOT _location)
        message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares")
      endif ()
-      add_library (c-ares::cares ${_static} IMPORTED GLOBAL)
+      if(${_location} MATCHES "\\.a$")
+        add_library(c-ares::cares STATIC IMPORTED GLOBAL)
+      else()
+        add_library(c-ares::cares SHARED IMPORTED GLOBAL)
+      endif()      
      set_target_properties (c-ares::cares PROPERTIES
        IMPORTED_LOCATION ${_location}
        INTERFACE_INCLUDE_DIRECTORIES "${${_prefix}_INCLUDE_DIRS}"
@@ -204,6 +208,7 @@ else ()
      CMAKE_ARGS
        -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
        -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+        -DCMAKE_CXX_STANDARD=17
        $<$<BOOL:${CMAKE_VERBOSE_MAKEFILE}>:-DCMAKE_VERBOSE_MAKEFILE=ON>
        $<$<BOOL:${CMAKE_TOOLCHAIN_FILE}>:-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}>
        $<$<BOOL:${VCPKG_TARGET_TRIPLET}>:-DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}>
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,14 +1,18 @@
 cmake_minimum_required (VERSION 3.16)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+

 if (POLICY CMP0074)
  cmake_policy(SET CMP0074 NEW)
 endif ()

-project (rippled)
-set(CMAKE_CXX_EXTENSIONS OFF)
-set(CMAKE_CXX_STANDARD 20)
-set(CMAKE_CXX_STANDARD_REQUIRED ON)
+if(POLICY CMP0144)
+  cmake_policy(SET CMP0144 NEW)
+endif()

+project (rippled)
 set(Boost_NO_BOOST_CMAKE ON)

 # make GIT_COMMIT_HASH define available to all sources
--- a/src/ripple/protocol/jss.h
+++ b/src/ripple/protocol/jss.h
@@ -240,6 +240,7 @@ JSS(complete);               // out: NetworkOPs, InboundLedger
 JSS(complete_ledgers);       // out: NetworkOPs, PeerImp
 JSS(complete_ledgers_pinned);
 JSS(complete_shards);  // out: OverlayImpl, PeerImp
+JSS(compression_level);
 JSS(consensus);        // out: NetworkOPs, LedgerConsensus
 JSS(converge_time);    // out: NetworkOPs
 JSS(converge_time_s);  // out: NetworkOPs
@@ -401,25 +402,26 @@ JSS(ledger);                // in: NetworkOPs, LedgerCleaner,
 JSS(ledger_count);
 JSS(ledgers_loaded);
 JSS(ledgers_written);
-JSS(ledger_current_index);        // out: NetworkOPs, RPCHelpers,
-                                  //      LedgerCurrent, LedgerAccept,
-                                  //      AccountLines
-JSS(ledger_data);                 // out: LedgerHeader
-JSS(ledger_hash);                 // in: RPCHelpers, LedgerRequest,
-                                  //     RipplePathFind, TransactionEntry,
-                                  //     handlers/Ledger
-                                  // out: NetworkOPs, RPCHelpers,
-                                  //      LedgerClosed, LedgerData,
-                                  //      AccountLines
-JSS(ledger_hit_rate);             // out: GetCounts
-JSS(ledger_index);                // in/out: many
-JSS(ledger_index_max);            // in, out: AccountTx*
-JSS(ledger_index_min);            // in, out: AccountTx*
-JSS(ledger_max);                  // in, out: AccountTx*
-JSS(ledger_min);                  // in, out: AccountTx*
-JSS(ledger_time);                 // out: NetworkOPs
-JSS(LEDGER_ENTRY_TYPES);          // out: RPC server_definitions
-JSS(levels);                      // LogLevels
+JSS(ledger_current_index);  // out: NetworkOPs, RPCHelpers,
+                            //      LedgerCurrent, LedgerAccept,
+                            //      AccountLines
+JSS(ledger_data);           // out: LedgerHeader
+JSS(ledger_hash);           // in: RPCHelpers, LedgerRequest,
+                            //     RipplePathFind, TransactionEntry,
+                            //     handlers/Ledger
+                            // out: NetworkOPs, RPCHelpers,
+                            //      LedgerClosed, LedgerData,
+                            //      AccountLines
+JSS(ledger_hit_rate);       // out: GetCounts
+JSS(ledger_index);          // in/out: many
+JSS(ledger_index_max);      // in, out: AccountTx*
+JSS(ledger_index_min);      // in, out: AccountTx*
+JSS(ledger_max);            // in, out: AccountTx*
+JSS(ledger_min);            // in, out: AccountTx*
+JSS(ledger_time);           // out: NetworkOPs
+JSS(LEDGER_ENTRY_TYPES);    // out: RPC server_definitions
+JSS(levels);                // LogLevels
+JSS(level);
 JSS(limit);                       // in/out: AccountTx*, AccountOffers,
                                  //         AccountLines, AccountObjects
                                  // in: LedgerData, BookOffers
--- a/src/ripple/rpc/handlers/Catalogue.cpp
+++ b/src/ripple/rpc/handlers/Catalogue.cpp
@@ -46,14 +46,59 @@
 #include <unordered_set>
 #include <vector>

+#include <boost/iostreams/filter/zlib.hpp>
+#include <boost/iostreams/filtering_stream.hpp>
+
 namespace ripple {

 using time_point = NetClock::time_point;
 using duration = NetClock::duration;

-static constexpr uint16_t CATALOGUE_VERSION = 1;
 #define CATL 0x4C544143UL /*"CATL" in LE*/

+// Replace the current version constant
+static constexpr uint16_t CATALOGUE_VERSION = 1;
+
+// Instead use these definitions
+static constexpr uint16_t CATALOGUE_VERSION_MASK =
+    0x00FF;  // Lower 8 bits for version
+static constexpr uint16_t CATALOGUE_COMPRESS_LEVEL_MASK =
+    0x0F00;  // Bits 8-11: compression level
+static constexpr uint16_t CATALOGUE_RESERVED_MASK =
+    0xF000;  // Bits 12-15: reserved
+
+// Helper functions for version field manipulation
+inline uint8_t
+getCatalogueVersion(uint16_t versionField)
+{
+    return versionField & CATALOGUE_VERSION_MASK;
+}
+
+inline uint8_t
+getCompressionLevel(uint16_t versionField)
+{
+    return (versionField & CATALOGUE_COMPRESS_LEVEL_MASK) >> 8;
+}
+
+inline bool
+isCompressed(uint16_t versionField)
+{
+    return getCompressionLevel(versionField) > 0;
+}
+
+inline uint16_t
+makeCatalogueVersionField(uint8_t version, uint8_t compressionLevel = 0)
+{  // 0 = no compression
+
+    // Ensure compression level is within valid range (0-9)
+    if (compressionLevel > 9)
+        compressionLevel = 9;
+
+    uint16_t result = version & CATALOGUE_VERSION_MASK;
+    result |= (compressionLevel << 8);  // Store level in bits 8-11
+    return result;
+}
+
 #pragma pack(push, 1)  // pack the struct tightly
 struct CATLHeader
 {
@@ -82,6 +127,32 @@ doCatalogueCreate(RPC::JsonContext& context)
            rpcINVALID_PARAMS,
            "expected output_file: <absolute writeable filepath>");

+    uint8_t compressionLevel = 0;  // Default: no compression
+
+    if (context.params.isMember(jss::compression_level))
+    {
+        if (context.params[jss::compression_level].isObject())
+        {
+            auto const& comp = context.params[jss::compression_level];
+
+            if (comp.isMember(jss::level))
+            {
+                compressionLevel = comp[jss::level].asUInt();
+                if (compressionLevel > 9)
+                    compressionLevel = 9;
+            }
+            else
+            {
+                compressionLevel =
+                    6;  // Default level if compression is enabled
+            }
+        }
+        else if (context.params[jss::compression_level].asBool())
+        {
+            compressionLevel = 6;  // Default level if compression is enabled
+        }
+    }
+
    // Check output file isn't already populated and can be written to
    {
        struct stat st;
@@ -137,7 +208,8 @@ doCatalogueCreate(RPC::JsonContext& context)
    CATLHeader header;
    header.min_ledger = min_ledger;
    header.max_ledger = max_ledger;
-    header.version = CATALOGUE_VERSION;
+    header.version =
+        makeCatalogueVersionField(CATALOGUE_VERSION, compressionLevel);
    header.network_id = context.app.config().NETWORK_ID;

    outfile.write(reinterpret_cast<const char*>(&header), sizeof(CATLHeader));
@@ -146,10 +218,15 @@ doCatalogueCreate(RPC::JsonContext& context)
            rpcINTERNAL,
            "failed to write header: " + std::string(strerror(errno)));

+    auto compStream = std::make_unique<boost::iostreams::filtering_ostream>();
+    compStream->push(boost::iostreams::zlib_compressor(
+        boost::iostreams::zlib_params(compressionLevel)));
+    compStream->push(boost::ref(outfile));
+
    // Process ledgers with local processor implementation
-    auto writeToFile = [&outfile, &context](const void* data, size_t size) {
-        outfile.write(reinterpret_cast<const char*>(data), size);
-        if (outfile.fail())
+    auto writeToFile = [&compStream, &context](const void* data, size_t size) {
+        compStream->write(reinterpret_cast<const char*>(data), size);
+        if (compStream->fail())
        {
            JLOG(context.j.error())
                << "Failed to write to output file: " << std::strerror(errno);
@@ -159,7 +236,7 @@ doCatalogueCreate(RPC::JsonContext& context)
    };

    auto outputLedger =
-        [&writeToFile, &ledgers, &context, &outfile](
+        [&writeToFile, &ledgers, &context, &compStream](
            uint32_t seq,
            std::optional<std::reference_wrapper<const SHAMap>> prevStateMap =
                std::nullopt) -> bool {
@@ -203,8 +280,9 @@ doCatalogueCreate(RPC::JsonContext& context)
            }

            size_t stateNodesWritten =
-                ledger->stateMap().serializeToStream(outfile, prevStateMap);
-            size_t txNodesWritten = ledger->txMap().serializeToStream(outfile);
+                ledger->stateMap().serializeToStream(*compStream, prevStateMap);
+            size_t txNodesWritten =
+                ledger->txMap().serializeToStream(*compStream);

            JLOG(context.j.info()) << "Ledger " << seq << ": Wrote "
                                   << stateNodesWritten << " state nodes, "
@@ -242,8 +320,13 @@ doCatalogueCreate(RPC::JsonContext& context)
                rpcINTERNAL, "Error occurred while processing ledgers");
    }

-    // Get the final file size
+    // flush and finish
+    compStream->flush();
+    compStream->reset();
    outfile.flush();
+    outfile.close();
+
+    // Get the final file size
    struct stat st;
    if (stat(filepath.c_str(), &st) != 0)
    {
@@ -261,6 +344,7 @@ doCatalogueCreate(RPC::JsonContext& context)
    jvResult[jss::file_size] = (Json::UInt)(file_size);
    jvResult[jss::ledgers_written] = static_cast<Json::UInt>(ledgers_written);
    jvResult[jss::status] = jss::success;
+    jvResult[jss::compression_level] = compressionLevel;

    return jvResult;
 }
@@ -316,15 +400,18 @@ doCatalogueLoad(RPC::JsonContext& context)
    if (header.magic != CATL)
        return rpcError(rpcINVALID_PARAMS, "invalid catalogue file magic");

-    JLOG(context.j.info()) << "Catalogue version: " << header.version;
-    JLOG(context.j.info()) << "Ledger range: " << header.min_ledger << " - "
-                           << header.max_ledger;
-    JLOG(context.j.info()) << "Network ID: " << header.network_id;
+    // Extract version information
+    uint8_t version = getCatalogueVersion(header.version);
+    uint8_t compressionLevel = getCompressionLevel(header.version);

-    if (header.version != CATALOGUE_VERSION)
+    JLOG(context.j.info()) << "Catalogue version: " << (int)version;
+    JLOG(context.j.info()) << "Compression level: " << (int)compressionLevel;
+
+    // Check version compatibility
+    if (version > 1)  // Only checking base version number
        return rpcError(
            rpcINVALID_PARAMS,
-            "unsupported catalogue version: " + std::to_string(header.version));
+            "unsupported catalogue version: " + std::to_string(version));

    if (header.network_id != context.app.config().NETWORK_ID)
        return rpcError(
@@ -332,12 +419,17 @@ doCatalogueLoad(RPC::JsonContext& context)
            "catalogue network ID mismatch: " +
                std::to_string(header.network_id));

+    // Set up decompression if needed
+    auto decompStream = std::make_unique<boost::iostreams::filtering_istream>();
+    decompStream->push(boost::iostreams::zlib_decompressor());
+    decompStream->push(boost::ref(infile));
+
    uint32_t ledgersLoaded = 0;
    std::shared_ptr<Ledger> prevLedger;
    uint32_t expected_seq = header.min_ledger;

    // Process each ledger sequentially
-    while (!infile.eof() && expected_seq <= header.max_ledger)
+    while (!decompStream->eof() && expected_seq <= header.max_ledger)
    {
        LedgerInfo info;
        uint64_t closeTime = -1;
@@ -345,23 +437,27 @@ doCatalogueLoad(RPC::JsonContext& context)
        uint32_t closeTimeResolution = -1;
        uint64_t drops = -1;

-        if (!infile.read(
+        if (!decompStream->read(
                reinterpret_cast<char*>(&info.seq), sizeof(info.seq)) ||
-            !infile.read(reinterpret_cast<char*>(info.hash.data()), 32) ||
-            !infile.read(reinterpret_cast<char*>(info.txHash.data()), 32) ||
-            !infile.read(
+            !decompStream->read(
+                reinterpret_cast<char*>(info.hash.data()), 32) ||
+            !decompStream->read(
+                reinterpret_cast<char*>(info.txHash.data()), 32) ||
+            !decompStream->read(
                reinterpret_cast<char*>(info.accountHash.data()), 32) ||
-            !infile.read(reinterpret_cast<char*>(info.parentHash.data()), 32) ||
-            !infile.read(reinterpret_cast<char*>(&drops), sizeof(drops)) ||
-            !infile.read(
+            !decompStream->read(
+                reinterpret_cast<char*>(info.parentHash.data()), 32) ||
+            !decompStream->read(
+                reinterpret_cast<char*>(&drops), sizeof(drops)) ||
+            !decompStream->read(
                reinterpret_cast<char*>(&info.closeFlags),
                sizeof(info.closeFlags)) ||
-            !infile.read(
+            !decompStream->read(
                reinterpret_cast<char*>(&closeTimeResolution),
                sizeof(closeTimeResolution)) ||
-            !infile.read(
+            !decompStream->read(
                reinterpret_cast<char*>(&closeTime), sizeof(closeTime)) ||
-            !infile.read(
+            !decompStream->read(
                reinterpret_cast<char*>(&parentCloseTime),
                sizeof(parentCloseTime)))
        {
@@ -401,7 +497,7 @@ doCatalogueLoad(RPC::JsonContext& context)
            ledger->setLedgerInfo(info);

            // Deserialize the complete state map from leaf nodes
-            if (!ledger->stateMap().deserializeFromStream(infile))
+            if (!ledger->stateMap().deserializeFromStream(*decompStream))
            {
                JLOG(context.j.error())
                    << "Failed to deserialize base ledger state";
@@ -427,7 +523,7 @@ doCatalogueLoad(RPC::JsonContext& context)
                *snapshot);

            // Apply delta (only leaf-node changes)
-            if (!ledger->stateMap().deserializeFromStream(infile))
+            if (!ledger->stateMap().deserializeFromStream(*decompStream))
            {
                JLOG(context.j.error())
                    << "Failed to apply delta to ledger " << info.seq;
@@ -436,7 +532,7 @@ doCatalogueLoad(RPC::JsonContext& context)
        }

        // pull in the tx map
-        if (!ledger->txMap().deserializeFromStream(infile))
+        if (!ledger->txMap().deserializeFromStream(*decompStream))
        {
            JLOG(context.j.error())
                << "Failed to apply delta to ledger " << info.seq;
@@ -457,8 +553,7 @@ doCatalogueLoad(RPC::JsonContext& context)
        ledger->setImmutable(true);

        // Save in database
-        std::cout << "pendSaveValidated\n";
-        pendSaveValidated(context.app, ledger, true, false);
+        pendSaveValidated(context.app, ledger, false, false);

        // Store in ledger master
        context.app.getLedgerMaster().storeLedger(ledger, true);
@@ -476,7 +571,7 @@ doCatalogueLoad(RPC::JsonContext& context)
        ledgersLoaded++;
    }

-    // Close the file as we've read all the data
+    decompStream->reset();
    infile.close();

    // Update ledger range in ledger master
@@ -495,6 +590,7 @@ doCatalogueLoad(RPC::JsonContext& context)
    jvResult[jss::ledgers_loaded] = static_cast<Json::UInt>(ledgersLoaded);
    jvResult[jss::file_size] = (Json::UInt)(file_size);
    jvResult[jss::status] = jss::success;
+    jvResult[jss::compression_level] = compressionLevel;

    return jvResult;
 }