diff --git a/Builds/CMake/deps/Boost.cmake b/Builds/CMake/deps/Boost.cmake index 5038234bc..6469ba15d 100644 --- a/Builds/CMake/deps/Boost.cmake +++ b/Builds/CMake/deps/Boost.cmake @@ -1,14 +1,16 @@ #[===================================================================[ NIH dep: boost #]===================================================================] - if((NOT DEFINED BOOST_ROOT) AND(DEFINED ENV{BOOST_ROOT})) set(BOOST_ROOT $ENV{BOOST_ROOT}) endif() +if((NOT DEFINED BOOST_LIBRARYDIR) AND(DEFINED ENV{BOOST_LIBRARYDIR})) + set(BOOST_LIBRARYDIR $ENV{BOOST_LIBRARYDIR}) +endif() file(TO_CMAKE_PATH "${BOOST_ROOT}" BOOST_ROOT) if(WIN32 OR CYGWIN) # Workaround for MSVC having two boost versions - x86 and x64 on same PC in stage folders - if(DEFINED BOOST_ROOT) + if((NOT DEFINED BOOST_LIBRARYDIR) AND (DEFINED BOOST_ROOT)) if(IS_DIRECTORY ${BOOST_ROOT}/stage64/lib) set(BOOST_LIBRARYDIR ${BOOST_ROOT}/stage64/lib) elseif(IS_DIRECTORY ${BOOST_ROOT}/stage/lib) @@ -55,6 +57,7 @@ find_package(Boost 1.86 REQUIRED program_options regex system + iostreams thread) add_library(ripple_boost INTERFACE) @@ -74,6 +77,7 @@ target_link_libraries(ripple_boost Boost::coroutine Boost::date_time Boost::filesystem + Boost::iostreams Boost::program_options Boost::regex Boost::system diff --git a/Builds/CMake/deps/FindBoost.cmake b/Builds/CMake/deps/FindBoost.cmake index 121e72641..e2b921ca7 100644 --- a/Builds/CMake/deps/FindBoost.cmake +++ b/Builds/CMake/deps/FindBoost.cmake @@ -248,6 +248,7 @@ include(FindPackageHandleStandardArgs) # Save project's policies cmake_policy(PUSH) cmake_policy(SET CMP0057 NEW) # if IN_LIST +cmake_policy(SET CMP0144 NEW) #------------------------------------------------------------------------------- # Before we go searching, check whether a boost cmake package is available, unless @@ -969,7 +970,24 @@ function(_Boost_COMPONENT_DEPENDENCIES component _ret) set(_Boost_WAVE_DEPENDENCIES filesystem serialization thread chrono date_time atomic) set(_Boost_WSERIALIZATION_DEPENDENCIES serialization) endif() - if(NOT Boost_VERSION_STRING VERSION_LESS 1.77.0) + + # Special handling for Boost 1.86.0 and higher + if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0) + # Explicitly set these for Boost 1.86 + set(_Boost_IOSTREAMS_DEPENDENCIES "") # No dependencies for iostreams in 1.86 + + # Debug output to help diagnose the issue + if(Boost_DEBUG) + message(STATUS "Using special dependency settings for Boost 1.86.0+") + message(STATUS "Component: ${component}, uppercomponent: ${uppercomponent}") + message(STATUS "Boost_VERSION_STRING: ${Boost_VERSION_STRING}") + message(STATUS "BOOST_ROOT: $ENV{BOOST_ROOT}") + message(STATUS "BOOST_LIBRARYDIR: $ENV{BOOST_LIBRARYDIR}") + endif() + endif() + + # Only show warning for versions beyond what we've defined + if(NOT Boost_VERSION_STRING VERSION_LESS 1.87.0) message(WARNING "New Boost version may have incorrect or missing dependencies and imported targets") endif() endif() @@ -1879,6 +1897,18 @@ foreach(COMPONENT ${Boost_FIND_COMPONENTS}) list(INSERT _boost_LIBRARY_SEARCH_DIRS_RELEASE 0 ${Boost_LIBRARY_DIR_DEBUG}) endif() + if(NOT Boost_VERSION_STRING VERSION_LESS 1.86.0) + if(BOOST_LIBRARYDIR AND EXISTS "${BOOST_LIBRARYDIR}") + # Clear existing search paths and use only BOOST_LIBRARYDIR + set(_boost_LIBRARY_SEARCH_DIRS_RELEASE "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH) + set(_boost_LIBRARY_SEARCH_DIRS_DEBUG "${BOOST_LIBRARYDIR}" NO_DEFAULT_PATH) + + if(Boost_DEBUG) + message(STATUS "Boost 1.86: Setting library search dirs to BOOST_LIBRARYDIR: ${BOOST_LIBRARYDIR}") + endif() + endif() + endif() + # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS_RELEASE}") diff --git a/Builds/CMake/deps/gRPC.cmake b/Builds/CMake/deps/gRPC.cmake index 8dd094175..e4beaf89d 100644 --- a/Builds/CMake/deps/gRPC.cmake +++ b/Builds/CMake/deps/gRPC.cmake @@ -74,7 +74,11 @@ else () if (NOT _location) message (FATAL_ERROR "using pkg-config for grpc, can't find c-ares") endif () - add_library (c-ares::cares ${_static} IMPORTED GLOBAL) + if(${_location} MATCHES "\\.a$") + add_library(c-ares::cares STATIC IMPORTED GLOBAL) + else() + add_library(c-ares::cares SHARED IMPORTED GLOBAL) + endif() set_target_properties (c-ares::cares PROPERTIES IMPORTED_LOCATION ${_location} INTERFACE_INCLUDE_DIRECTORIES "${${_prefix}_INCLUDE_DIRS}" @@ -204,6 +208,7 @@ else () CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_STANDARD=17 $<$:-DCMAKE_VERBOSE_MAKEFILE=ON> $<$:-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}> $<$:-DVCPKG_TARGET_TRIPLET=${VCPKG_TARGET_TRIPLET}> diff --git a/CMakeLists.txt b/CMakeLists.txt index d62541fad..25b530328 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,14 +1,18 @@ cmake_minimum_required (VERSION 3.16) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + if (POLICY CMP0074) cmake_policy(SET CMP0074 NEW) endif () -project (rippled) -set(CMAKE_CXX_EXTENSIONS OFF) -set(CMAKE_CXX_STANDARD 20) -set(CMAKE_CXX_STANDARD_REQUIRED ON) +if(POLICY CMP0144) + cmake_policy(SET CMP0144 NEW) +endif() +project (rippled) set(Boost_NO_BOOST_CMAKE ON) # make GIT_COMMIT_HASH define available to all sources diff --git a/src/ripple/protocol/jss.h b/src/ripple/protocol/jss.h index 6efd36a36..1a2cf8568 100644 --- a/src/ripple/protocol/jss.h +++ b/src/ripple/protocol/jss.h @@ -240,6 +240,7 @@ JSS(complete); // out: NetworkOPs, InboundLedger JSS(complete_ledgers); // out: NetworkOPs, PeerImp JSS(complete_ledgers_pinned); JSS(complete_shards); // out: OverlayImpl, PeerImp +JSS(compression_level); JSS(consensus); // out: NetworkOPs, LedgerConsensus JSS(converge_time); // out: NetworkOPs JSS(converge_time_s); // out: NetworkOPs @@ -401,25 +402,26 @@ JSS(ledger); // in: NetworkOPs, LedgerCleaner, JSS(ledger_count); JSS(ledgers_loaded); JSS(ledgers_written); -JSS(ledger_current_index); // out: NetworkOPs, RPCHelpers, - // LedgerCurrent, LedgerAccept, - // AccountLines -JSS(ledger_data); // out: LedgerHeader -JSS(ledger_hash); // in: RPCHelpers, LedgerRequest, - // RipplePathFind, TransactionEntry, - // handlers/Ledger - // out: NetworkOPs, RPCHelpers, - // LedgerClosed, LedgerData, - // AccountLines -JSS(ledger_hit_rate); // out: GetCounts -JSS(ledger_index); // in/out: many -JSS(ledger_index_max); // in, out: AccountTx* -JSS(ledger_index_min); // in, out: AccountTx* -JSS(ledger_max); // in, out: AccountTx* -JSS(ledger_min); // in, out: AccountTx* -JSS(ledger_time); // out: NetworkOPs -JSS(LEDGER_ENTRY_TYPES); // out: RPC server_definitions -JSS(levels); // LogLevels +JSS(ledger_current_index); // out: NetworkOPs, RPCHelpers, + // LedgerCurrent, LedgerAccept, + // AccountLines +JSS(ledger_data); // out: LedgerHeader +JSS(ledger_hash); // in: RPCHelpers, LedgerRequest, + // RipplePathFind, TransactionEntry, + // handlers/Ledger + // out: NetworkOPs, RPCHelpers, + // LedgerClosed, LedgerData, + // AccountLines +JSS(ledger_hit_rate); // out: GetCounts +JSS(ledger_index); // in/out: many +JSS(ledger_index_max); // in, out: AccountTx* +JSS(ledger_index_min); // in, out: AccountTx* +JSS(ledger_max); // in, out: AccountTx* +JSS(ledger_min); // in, out: AccountTx* +JSS(ledger_time); // out: NetworkOPs +JSS(LEDGER_ENTRY_TYPES); // out: RPC server_definitions +JSS(levels); // LogLevels +JSS(level); JSS(limit); // in/out: AccountTx*, AccountOffers, // AccountLines, AccountObjects // in: LedgerData, BookOffers diff --git a/src/ripple/rpc/handlers/Catalogue.cpp b/src/ripple/rpc/handlers/Catalogue.cpp index eeee3e551..03f1f1dfb 100644 --- a/src/ripple/rpc/handlers/Catalogue.cpp +++ b/src/ripple/rpc/handlers/Catalogue.cpp @@ -46,14 +46,59 @@ #include #include +#include +#include + namespace ripple { using time_point = NetClock::time_point; using duration = NetClock::duration; -static constexpr uint16_t CATALOGUE_VERSION = 1; #define CATL 0x4C544143UL /*"CATL" in LE*/ +// Replace the current version constant +static constexpr uint16_t CATALOGUE_VERSION = 1; + +// Instead use these definitions +static constexpr uint16_t CATALOGUE_VERSION_MASK = + 0x00FF; // Lower 8 bits for version +static constexpr uint16_t CATALOGUE_COMPRESS_LEVEL_MASK = + 0x0F00; // Bits 8-11: compression level +static constexpr uint16_t CATALOGUE_RESERVED_MASK = + 0xF000; // Bits 12-15: reserved + +// Helper functions for version field manipulation +inline uint8_t +getCatalogueVersion(uint16_t versionField) +{ + return versionField & CATALOGUE_VERSION_MASK; +} + +inline uint8_t +getCompressionLevel(uint16_t versionField) +{ + return (versionField & CATALOGUE_COMPRESS_LEVEL_MASK) >> 8; +} + +inline bool +isCompressed(uint16_t versionField) +{ + return getCompressionLevel(versionField) > 0; +} + +inline uint16_t +makeCatalogueVersionField(uint8_t version, uint8_t compressionLevel = 0) +{ // 0 = no compression + + // Ensure compression level is within valid range (0-9) + if (compressionLevel > 9) + compressionLevel = 9; + + uint16_t result = version & CATALOGUE_VERSION_MASK; + result |= (compressionLevel << 8); // Store level in bits 8-11 + return result; +} + #pragma pack(push, 1) // pack the struct tightly struct CATLHeader { @@ -82,6 +127,32 @@ doCatalogueCreate(RPC::JsonContext& context) rpcINVALID_PARAMS, "expected output_file: "); + uint8_t compressionLevel = 0; // Default: no compression + + if (context.params.isMember(jss::compression_level)) + { + if (context.params[jss::compression_level].isObject()) + { + auto const& comp = context.params[jss::compression_level]; + + if (comp.isMember(jss::level)) + { + compressionLevel = comp[jss::level].asUInt(); + if (compressionLevel > 9) + compressionLevel = 9; + } + else + { + compressionLevel = + 6; // Default level if compression is enabled + } + } + else if (context.params[jss::compression_level].asBool()) + { + compressionLevel = 6; // Default level if compression is enabled + } + } + // Check output file isn't already populated and can be written to { struct stat st; @@ -137,7 +208,8 @@ doCatalogueCreate(RPC::JsonContext& context) CATLHeader header; header.min_ledger = min_ledger; header.max_ledger = max_ledger; - header.version = CATALOGUE_VERSION; + header.version = + makeCatalogueVersionField(CATALOGUE_VERSION, compressionLevel); header.network_id = context.app.config().NETWORK_ID; outfile.write(reinterpret_cast(&header), sizeof(CATLHeader)); @@ -146,10 +218,15 @@ doCatalogueCreate(RPC::JsonContext& context) rpcINTERNAL, "failed to write header: " + std::string(strerror(errno))); + auto compStream = std::make_unique(); + compStream->push(boost::iostreams::zlib_compressor( + boost::iostreams::zlib_params(compressionLevel))); + compStream->push(boost::ref(outfile)); + // Process ledgers with local processor implementation - auto writeToFile = [&outfile, &context](const void* data, size_t size) { - outfile.write(reinterpret_cast(data), size); - if (outfile.fail()) + auto writeToFile = [&compStream, &context](const void* data, size_t size) { + compStream->write(reinterpret_cast(data), size); + if (compStream->fail()) { JLOG(context.j.error()) << "Failed to write to output file: " << std::strerror(errno); @@ -159,7 +236,7 @@ doCatalogueCreate(RPC::JsonContext& context) }; auto outputLedger = - [&writeToFile, &ledgers, &context, &outfile]( + [&writeToFile, &ledgers, &context, &compStream]( uint32_t seq, std::optional> prevStateMap = std::nullopt) -> bool { @@ -203,8 +280,9 @@ doCatalogueCreate(RPC::JsonContext& context) } size_t stateNodesWritten = - ledger->stateMap().serializeToStream(outfile, prevStateMap); - size_t txNodesWritten = ledger->txMap().serializeToStream(outfile); + ledger->stateMap().serializeToStream(*compStream, prevStateMap); + size_t txNodesWritten = + ledger->txMap().serializeToStream(*compStream); JLOG(context.j.info()) << "Ledger " << seq << ": Wrote " << stateNodesWritten << " state nodes, " @@ -242,8 +320,13 @@ doCatalogueCreate(RPC::JsonContext& context) rpcINTERNAL, "Error occurred while processing ledgers"); } - // Get the final file size + // flush and finish + compStream->flush(); + compStream->reset(); outfile.flush(); + outfile.close(); + + // Get the final file size struct stat st; if (stat(filepath.c_str(), &st) != 0) { @@ -261,6 +344,7 @@ doCatalogueCreate(RPC::JsonContext& context) jvResult[jss::file_size] = (Json::UInt)(file_size); jvResult[jss::ledgers_written] = static_cast(ledgers_written); jvResult[jss::status] = jss::success; + jvResult[jss::compression_level] = compressionLevel; return jvResult; } @@ -316,15 +400,18 @@ doCatalogueLoad(RPC::JsonContext& context) if (header.magic != CATL) return rpcError(rpcINVALID_PARAMS, "invalid catalogue file magic"); - JLOG(context.j.info()) << "Catalogue version: " << header.version; - JLOG(context.j.info()) << "Ledger range: " << header.min_ledger << " - " - << header.max_ledger; - JLOG(context.j.info()) << "Network ID: " << header.network_id; + // Extract version information + uint8_t version = getCatalogueVersion(header.version); + uint8_t compressionLevel = getCompressionLevel(header.version); - if (header.version != CATALOGUE_VERSION) + JLOG(context.j.info()) << "Catalogue version: " << (int)version; + JLOG(context.j.info()) << "Compression level: " << (int)compressionLevel; + + // Check version compatibility + if (version > 1) // Only checking base version number return rpcError( rpcINVALID_PARAMS, - "unsupported catalogue version: " + std::to_string(header.version)); + "unsupported catalogue version: " + std::to_string(version)); if (header.network_id != context.app.config().NETWORK_ID) return rpcError( @@ -332,12 +419,17 @@ doCatalogueLoad(RPC::JsonContext& context) "catalogue network ID mismatch: " + std::to_string(header.network_id)); + // Set up decompression if needed + auto decompStream = std::make_unique(); + decompStream->push(boost::iostreams::zlib_decompressor()); + decompStream->push(boost::ref(infile)); + uint32_t ledgersLoaded = 0; std::shared_ptr prevLedger; uint32_t expected_seq = header.min_ledger; // Process each ledger sequentially - while (!infile.eof() && expected_seq <= header.max_ledger) + while (!decompStream->eof() && expected_seq <= header.max_ledger) { LedgerInfo info; uint64_t closeTime = -1; @@ -345,23 +437,27 @@ doCatalogueLoad(RPC::JsonContext& context) uint32_t closeTimeResolution = -1; uint64_t drops = -1; - if (!infile.read( + if (!decompStream->read( reinterpret_cast(&info.seq), sizeof(info.seq)) || - !infile.read(reinterpret_cast(info.hash.data()), 32) || - !infile.read(reinterpret_cast(info.txHash.data()), 32) || - !infile.read( + !decompStream->read( + reinterpret_cast(info.hash.data()), 32) || + !decompStream->read( + reinterpret_cast(info.txHash.data()), 32) || + !decompStream->read( reinterpret_cast(info.accountHash.data()), 32) || - !infile.read(reinterpret_cast(info.parentHash.data()), 32) || - !infile.read(reinterpret_cast(&drops), sizeof(drops)) || - !infile.read( + !decompStream->read( + reinterpret_cast(info.parentHash.data()), 32) || + !decompStream->read( + reinterpret_cast(&drops), sizeof(drops)) || + !decompStream->read( reinterpret_cast(&info.closeFlags), sizeof(info.closeFlags)) || - !infile.read( + !decompStream->read( reinterpret_cast(&closeTimeResolution), sizeof(closeTimeResolution)) || - !infile.read( + !decompStream->read( reinterpret_cast(&closeTime), sizeof(closeTime)) || - !infile.read( + !decompStream->read( reinterpret_cast(&parentCloseTime), sizeof(parentCloseTime))) { @@ -401,7 +497,7 @@ doCatalogueLoad(RPC::JsonContext& context) ledger->setLedgerInfo(info); // Deserialize the complete state map from leaf nodes - if (!ledger->stateMap().deserializeFromStream(infile)) + if (!ledger->stateMap().deserializeFromStream(*decompStream)) { JLOG(context.j.error()) << "Failed to deserialize base ledger state"; @@ -427,7 +523,7 @@ doCatalogueLoad(RPC::JsonContext& context) *snapshot); // Apply delta (only leaf-node changes) - if (!ledger->stateMap().deserializeFromStream(infile)) + if (!ledger->stateMap().deserializeFromStream(*decompStream)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; @@ -436,7 +532,7 @@ doCatalogueLoad(RPC::JsonContext& context) } // pull in the tx map - if (!ledger->txMap().deserializeFromStream(infile)) + if (!ledger->txMap().deserializeFromStream(*decompStream)) { JLOG(context.j.error()) << "Failed to apply delta to ledger " << info.seq; @@ -457,8 +553,7 @@ doCatalogueLoad(RPC::JsonContext& context) ledger->setImmutable(true); // Save in database - std::cout << "pendSaveValidated\n"; - pendSaveValidated(context.app, ledger, true, false); + pendSaveValidated(context.app, ledger, false, false); // Store in ledger master context.app.getLedgerMaster().storeLedger(ledger, true); @@ -476,7 +571,7 @@ doCatalogueLoad(RPC::JsonContext& context) ledgersLoaded++; } - // Close the file as we've read all the data + decompStream->reset(); infile.close(); // Update ledger range in ledger master @@ -495,6 +590,7 @@ doCatalogueLoad(RPC::JsonContext& context) jvResult[jss::ledgers_loaded] = static_cast(ledgersLoaded); jvResult[jss::file_size] = (Json::UInt)(file_size); jvResult[jss::status] = jss::success; + jvResult[jss::compression_level] = compressionLevel; return jvResult; }