ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-30 00:25:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Read and write LedgerCache to file (#2761)

Browse Source 
Fixes #2413.
This commit is contained in:
Sergey Kuznetsov
2025-11-13 17:01:40 +00:00
committed by GitHub
parent c6308ce036
commit 346c9f9bdf
35 changed files with 2725 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
docs/config-description.md
View File

@@ -441,6 +441,22 @@ This document provides a list of all available Clio configuration properties in
- **Constraints**: The value must be one of the following: `sync`, `async`, `none`.
- **Description**: The strategy used for Cache loading.
### cache.file.path
- **Required**: False
- **Type**: string
- **Default value**: None
- **Constraints**: None
- **Description**: The path to a file where cache will be saved to on shutdown and loaded from on startup. If the file couldn't be read Clio will load cache as usual (from DB or from rippled).
### cache.file.max_sequence_age
- **Required**: True
- **Type**: int
- **Default value**: `5000`
- **Constraints**: None
- **Description**: Max allowed difference between the latest sequence in DB and in cache file. If the cache file is too old (contains too low latest sequence) Clio will reject using it.
### log.channels.[].channel
- **Required**: False

6
docs/examples/config/example-config.json
View File

@@ -137,7 +137,11 @@
// "num_cursors_from_account": 3200, // Read the cursors from the account table until we have enough cursors to partition the ledger to load concurrently.
"num_markers": 48, // The number of markers is the number of coroutines to load the cache concurrently.
"page_fetch_size": 512, // The number of rows to load for each page.
"load": "async" // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
"load": "async", // "sync" to load cache synchronously or "async" to load cache asynchronously or "none"/"no" to turn off the cache.
"file": {
"path": "./cache.bin",
"max_sequence_age": 5000
}
},
"prometheus": {
"enabled": true,

17
src/app/ClioApplication.cpp
View File

@@ -55,6 +55,7 @@
#include <cstdlib>
#include <memory>
#include <optional>
#include <string>
#include <thread>
#include <utility>
#include <vector>
@@ -110,7 +111,23 @@ ClioApplication::run(bool const useNgWebServer)
auto const dosguardWeights = web::dosguard::Weights::make(config_);
auto dosGuard = web::dosguard::DOSGuard{config_, whitelistHandler, dosguardWeights};
auto sweepHandler = web::dosguard::IntervalSweepHandler{config_, ioc, dosGuard};
auto cache = data::LedgerCache{};
appStopper_.setOnStop([&cache, this](auto&&) {
// TODO(kuznetsss): move this into Stopper::makeOnStopCallback()
auto const cacheFilePath = config_.maybeValue<std::string>("cache.file.path");
if (not cacheFilePath.has_value()) {
return;
}
LOG(util::LogService::info()) << "Saving ledger cache to " << *cacheFilePath;
if (auto const [success, duration_ms] = util::timed([&]() { return cache.saveToFile(*cacheFilePath); });
success.has_value()) {
LOG(util::LogService::info()) << "Successfully saved ledger cache in " << duration_ms << " ms";
} else {
LOG(util::LogService::error()) << "Error saving LedgerCache to file";
}
});
// Interface to the database
auto backend = data::makeBackend(config_, cache);

23
src/data/BackendInterface.cpp
View File

@@ -270,7 +270,7 @@ BackendInterface::updateRange(uint32_t newMax)
{
std::scoped_lock const lck(rngMtx_);
if (range_.has_value() && newMax < range_->maxSequence) {
if (range_.has_value() and newMax < range_->maxSequence) {
ASSERT(
false,
"Range shouldn't exist yet or newMax should be at least range->maxSequence. newMax = {}, "
@@ -280,11 +280,14 @@ BackendInterface::updateRange(uint32_t newMax)
);
}
if (!range_.has_value()) {
range_ = {.minSequence = newMax, .maxSequence = newMax};
} else {
range_->maxSequence = newMax;
updateRangeImpl(newMax);
}
void
BackendInterface::forceUpdateRange(uint32_t newMax)
{
std::scoped_lock const lck(rngMtx_);
updateRangeImpl(newMax);
}
void
@@ -410,4 +413,14 @@ BackendInterface::fetchFees(std::uint32_t const seq, boost::asio::yield_context
return fees;
}
void
BackendInterface::updateRangeImpl(uint32_t newMax)
{
if (!range_.has_value()) {
range_ = {.minSequence = newMax, .maxSequence = newMax};
} else {
range_->maxSequence = newMax;
}
}
} // namespace data

12
src/data/BackendInterface.hpp
View File

@@ -249,6 +249,15 @@ public:
void
updateRange(uint32_t newMax);
/**
* @brief Updates the range of sequences that are stored in the DB without any checks
* @note In the most cases you should use updateRange() instead
*
* @param newMax The new maximum sequence available
*/
void
forceUpdateRange(uint32_t newMax);
/**
* @brief Sets the range of sequences that are stored in the DB.
*
@@ -776,6 +785,9 @@ private:
*/
virtual bool
doFinishWrites() = 0;
void
updateRangeImpl(uint32_t newMax);
};
} // namespace data

3
src/data/CMakeLists.txt
View File

@@ -14,6 +14,9 @@ target_sources(
cassandra/impl/SslContext.cpp
cassandra/Handle.cpp
cassandra/SettingsProvider.cpp
impl/InputFile.cpp
impl/LedgerCacheFile.cpp
impl/OutputFile.cpp
)
target_link_libraries(clio_data PUBLIC cassandra-cpp-driver::cassandra-cpp-driver clio_util)

36
src/data/LedgerCache.cpp
View File

@@ -20,6 +20,7 @@
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include "data/impl/LedgerCacheFile.hpp"
#include "etl/Models.hpp"
#include "util/Assert.hpp"
@@ -27,9 +28,14 @@
#include <cstddef>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <map>
#include <mutex>
#include <optional>
#include <shared_mutex>
#include <string>
#include <utility>
#include <vector>
namespace data {
@@ -251,4 +257,34 @@ LedgerCache::getSuccessorHitRate() const
return static_cast<float>(successorHitCounter_.get().value()) / successorReqCounter_.get().value();
}
std::expected<void, std::string>
LedgerCache::saveToFile(std::string const& path) const
{
if (not isFull()) {
return std::unexpected{"Ledger cache is not full"};
}
impl::LedgerCacheFile file{path};
std::unique_lock lock{mtx_};
impl::LedgerCacheFile::DataView data{.latestSeq = latestSeq_, .map = map_, .deleted = deleted_};
return file.write(data);
}
std::expected<void, std::string>
LedgerCache::loadFromFile(std::string const& path, uint32_t minLatestSequence)
{
impl::LedgerCacheFile file{path};
auto data = file.read(minLatestSequence);
if (not data.has_value()) {
return std::unexpected(std::move(data).error());
}
auto [latestSeq, map, deleted] = std::move(data).value();
std::unique_lock lock{mtx_};
latestSeq_ = latestSeq;
map_ = std::move(map);
deleted_ = std::move(deleted);
full_ = true;
return {};
}
} // namespace data

23
src/data/LedgerCache.hpp
View File

@@ -37,6 +37,7 @@
#include <map>
#include <optional>
#include <shared_mutex>
#include <string>
#include <unordered_set>
#include <vector>
@@ -46,11 +47,16 @@ namespace data {
* @brief Cache for an entire ledger.
*/
class LedgerCache : public LedgerCacheInterface {
public:
/** @brief An entry of the cache */
struct CacheEntry {
uint32_t seq = 0;
Blob blob;
};
using CacheMap = std::map<ripple::uint256, CacheEntry>;
private:
// counters for fetchLedgerObject(s) hit rate
std::reference_wrapper<util::prometheus::CounterInt> objectReqCounter_{PrometheusService::counterInt(
"ledger_cache_counter_total_number",
@@ -73,8 +79,8 @@ class LedgerCache : public LedgerCacheInterface {
util::prometheus::Labels({{"type", "cache_hit"}, {"fetch", "successor_key"}})
)};
std::map<ripple::uint256, CacheEntry> map_;
std::map<ripple::uint256, CacheEntry> deleted_;
CacheMap map_;
CacheMap deleted_;
mutable std::shared_mutex mtx_;
std::condition_variable_any cv_;
@@ -138,6 +144,19 @@ public:
void
waitUntilCacheContainsSeq(uint32_t seq) override;
/**
* @brief Save the cache to file
* @note This operation takes about 7 seconds and it keeps mtx_ exclusively locked
*
* @param path The file path to save the cache to
* @return An error as a string if any
*/
std::expected<void, std::string>
saveToFile(std::string const& path) const;
std::expected<void, std::string>
loadFromFile(std::string const& path, uint32_t minLatestSequence) override;
};
} // namespace data

13
src/data/LedgerCacheInterface.hpp
View File

@@ -27,7 +27,9 @@
#include <cstddef>
#include <cstdint>
#include <expected>
#include <optional>
#include <string>
#include <vector>
namespace data {
@@ -168,6 +170,17 @@ public:
*/
virtual void
waitUntilCacheContainsSeq(uint32_t seq) = 0;
/**
* @brief Load the cache from file
* @note This operation takes about 7 seconds and it keeps mtx_ exclusively locked
*
* @param path The file path to load data from
* @param minLatestSequence The minimum allowed value of the latestLedgerSequence in cache file
* @return An error as a string if any
*/
[[nodiscard]] virtual std::expected<void, std::string>
loadFromFile(std::string const& path, uint32_t minLatestSequence) = 0;
};
} // namespace data

3
src/data/Types.hpp
View File

@@ -247,6 +247,9 @@ struct MPTHoldersAndCursor {
struct LedgerRange {
std::uint32_t minSequence = 0;
std::uint32_t maxSequence = 0;
bool
operator==(LedgerRange const&) const = default;
};
/**

58
src/data/impl/InputFile.cpp Normal file
View File

@@ -0,0 +1,58 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/impl/InputFile.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstring>
#include <ios>
#include <iosfwd>
#include <string>
#include <utility>
namespace data::impl {
InputFile::InputFile(std::string const& path) : file_(path, std::ios::binary | std::ios::in)
{
}
bool
InputFile::isOpen() const
{
return file_.is_open();
}
bool
InputFile::readRaw(char* data, size_t size)
{
file_.read(data, size);
shasum_.update(data, size);
return not file_.fail();
}
ripple::uint256
InputFile::hash() const
{
auto sum = shasum_;
return std::move(sum).finalize();
}
} // namespace data::impl

57
src/data/impl/InputFile.hpp Normal file
View File

@@ -0,0 +1,57 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "util/Shasum.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstring>
#include <fstream>
#include <iosfwd>
#include <string>
namespace data::impl {
class InputFile {
std::ifstream file_;
util::Sha256sum shasum_;
public:
InputFile(std::string const& path);
bool
isOpen() const;
template <typename T>
bool
read(T& t)
{
return readRaw(reinterpret_cast<char*>(&t), sizeof(T));
}
bool
readRaw(char* data, size_t size);
ripple::uint256
hash() const;
};
} // namespace data::impl

210
src/data/impl/LedgerCacheFile.cpp Normal file
View File

@@ -0,0 +1,210 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/impl/LedgerCacheFile.hpp"
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include <fmt/format.h>
#include <xrpl/basics/base_uint.h>
#include <algorithm>
#include <array>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <filesystem>
#include <string>
#include <utility>
namespace data::impl {
using Hash = ripple::uint256;
using Separator = std::array<char, 16>;
static constexpr Separator kSEPARATOR = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
namespace {
std::expected<std::pair<ripple::uint256, LedgerCache::CacheEntry>, std::string>
readCacheEntry(InputFile& file, size_t i)
{
ripple::uint256 key;
if (not file.readRaw(reinterpret_cast<char*>(key.data()), ripple::base_uint<256>::bytes)) {
return std::unexpected(fmt::format("Failed to read key at index {}", i));
}
uint32_t seq{};
if (not file.read(seq)) {
return std::unexpected(fmt::format("Failed to read sequence at index {}", i));
}
size_t blobSize{};
if (not file.read(blobSize)) {
return std::unexpected(fmt::format("Failed to read blob size at index {}", i));
}
Blob blob(blobSize);
if (not file.readRaw(reinterpret_cast<char*>(blob.data()), blobSize)) {
return std::unexpected(fmt::format("Failed to read blob data at index {}", i));
}
return std::make_pair(key, LedgerCache::CacheEntry{.seq = seq, .blob = std::move(blob)});
}
std::expected<void, std::string>
verifySeparator(Separator const& s)
{
if (not std::ranges::all_of(s, [](char c) { return c == 0; })) {
return std::unexpected{"Separator verification failed - data corruption detected"};
}
return {};
}
} // anonymous namespace
LedgerCacheFile::LedgerCacheFile(std::string path) : path_(std::move(path))
{
}
std::expected<void, std::string>
LedgerCacheFile::write(DataView dataView)
{
auto const newFilePath = fmt::format("{}.new", path_);
auto file = OutputFile{newFilePath};
if (not file.isOpen()) {
return std::unexpected{fmt::format("Couldn't open file: {}", newFilePath)};
}
Header const header{
.latestSeq = dataView.latestSeq, .mapSize = dataView.map.size(), .deletedSize = dataView.deleted.size()
};
file.write(header);
file.write(kSEPARATOR);
for (auto const& [k, v] : dataView.map) {
file.write(k.data(), decltype(k)::bytes);
file.write(v.seq);
file.write(v.blob.size());
file.writeRaw(reinterpret_cast<char const*>(v.blob.data()), v.blob.size());
}
file.write(kSEPARATOR);
for (auto const& [k, v] : dataView.deleted) {
file.write(k.data(), decltype(k)::bytes);
file.write(v.seq);
file.write(v.blob.size());
file.writeRaw(reinterpret_cast<char const*>(v.blob.data()), v.blob.size());
}
file.write(kSEPARATOR);
auto const hash = file.hash();
file.write(hash.data(), decltype(hash)::bytes);
try {
std::filesystem::rename(newFilePath, path_);
} catch (std::exception const& e) {
return std::unexpected{fmt::format("Error moving cache file from {} to {}: {}", newFilePath, path_, e.what())};
}
return {};
}
std::expected<LedgerCacheFile::Data, std::string>
LedgerCacheFile::read(uint32_t minLatestSequence)
{
try {
auto file = InputFile{path_};
if (not file.isOpen()) {
return std::unexpected{fmt::format("Couldn't open file: {}", path_)};
}
Data result;
Header header{};
if (not file.read(header)) {
return std::unexpected{"Error reading cache header"};
}
if (header.version != kVERSION) {
return std::unexpected{
fmt::format("Cache has wrong version: expected {} found {}", kVERSION, header.version)
};
}
if (header.latestSeq < minLatestSequence) {
return std::unexpected{fmt::format("Latest sequence ({}) in the cache file is too low.", header.latestSeq)};
}
result.latestSeq = header.latestSeq;
Separator separator{};
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading cache header"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
for (size_t i = 0; i < header.mapSize; ++i) {
auto cacheEntryExpected = readCacheEntry(file, i);
if (not cacheEntryExpected.has_value()) {
return std::unexpected{std::move(cacheEntryExpected).error()};
}
// Using insert with hint here to decrease insert operation complexity to the amortized constant instead of
// logN
result.map.insert(result.map.end(), std::move(cacheEntryExpected).value());
}
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading separator"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
for (size_t i = 0; i < header.deletedSize; ++i) {
auto cacheEntryExpected = readCacheEntry(file, i);
if (not cacheEntryExpected.has_value()) {
return std::unexpected{std::move(cacheEntryExpected).error()};
}
result.deleted.insert(result.deleted.end(), std::move(cacheEntryExpected).value());
}
if (not file.readRaw(separator.data(), separator.size())) {
return std::unexpected{"Error reading separator"};
}
if (auto verificationResult = verifySeparator(separator); not verificationResult.has_value()) {
return std::unexpected{std::move(verificationResult).error()};
}
auto const dataHash = file.hash();
ripple::uint256 hashFromFile{};
if (not file.readRaw(reinterpret_cast<char*>(hashFromFile.data()), decltype(hashFromFile)::bytes)) {
return std::unexpected{"Error reading hash"};
}
if (dataHash != hashFromFile) {
return std::unexpected{"Hash file corruption detected"};
}
return result;
} catch (std::exception const& e) {
return std::unexpected{fmt::format(" Error reading cache file: {}", e.what())};
} catch (...) {
return std::unexpected{fmt::format(" Error reading cache file")};
}
}
} // namespace data::impl

70
src/data/impl/LedgerCacheFile.hpp Normal file
View File

@@ -0,0 +1,70 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/LedgerCache.hpp"
#include "data/impl/InputFile.hpp"
#include "data/impl/OutputFile.hpp"
#include <fmt/format.h>
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <string>
namespace data::impl {
class LedgerCacheFile {
public:
struct Header {
uint32_t version = kVERSION;
uint32_t latestSeq{};
uint64_t mapSize{};
uint64_t deletedSize{};
};
private:
static constexpr uint32_t kVERSION = 1;
std::string path_;
public:
template <typename T>
struct DataBase {
uint32_t latestSeq{0};
T map;
T deleted;
};
using DataView = DataBase<LedgerCache::CacheMap const&>;
using Data = DataBase<LedgerCache::CacheMap>;
LedgerCacheFile(std::string path);
std::expected<void, std::string>
write(DataView dataView);
std::expected<Data, std::string>
read(uint32_t minLatestSequence);
};
} // namespace data::impl

62
src/data/impl/OutputFile.cpp Normal file
View File

@@ -0,0 +1,62 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/impl/OutputFile.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstring>
#include <ios>
#include <string>
#include <utility>
namespace data::impl {
OutputFile::OutputFile(std::string const& path) : file_(path, std::ios::binary | std::ios::out)
{
}
bool
OutputFile::isOpen() const
{
return file_.is_open();
}
void
OutputFile::writeRaw(char const* data, size_t size)
{
writeToFile(data, size);
}
void
OutputFile::writeToFile(char const* data, size_t size)
{
file_.write(data, size);
shasum_.update(data, size);
}
ripple::uint256
OutputFile::hash() const
{
auto sum = shasum_;
return std::move(sum).finalize();
}
} // namespace data::impl

68
src/data/impl/OutputFile.hpp Normal file
View File

@@ -0,0 +1,68 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "util/Shasum.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstring>
#include <fstream>
#include <string>
namespace data::impl {
class OutputFile {
std::ofstream file_;
util::Sha256sum shasum_;
public:
OutputFile(std::string const& path);
bool
isOpen() const;
template <typename T>
void
write(T&& data)
{
writeRaw(reinterpret_cast<char const*>(&data), sizeof(T));
}
template <typename T>
void
write(T const* data, size_t const size)
{
writeRaw(reinterpret_cast<char const*>(data), size);
}
void
writeRaw(char const* data, size_t size);
ripple::uint256
hash() const;
private:
void
writeToFile(char const* data, size_t size);
};
} // namespace data::impl

38
src/etl/CacheLoader.hpp
View File

@@ -21,6 +21,7 @@
#include "data/BackendInterface.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etl/CacheLoaderInterface.hpp"
#include "etl/CacheLoaderSettings.hpp"
#include "etl/impl/CacheLoader.hpp"
@@ -28,9 +29,12 @@
#include "etl/impl/CursorFromDiffProvider.hpp"
#include "etl/impl/CursorFromFixDiffNumProvider.hpp"
#include "util/Assert.hpp"
#include "util/Profiler.hpp"
#include "util/async/context/BasicExecutionContext.hpp"
#include "util/config/ConfigDefinition.hpp"
#include "util/log/Logger.hpp"
#include <algorithm>
#include <cstdint>
#include <functional>
#include <memory>
@@ -98,6 +102,10 @@ public:
return;
}
if (loadCacheFromFile()) {
return;
}
std::shared_ptr<impl::BaseCursorProvider> provider;
if (settings_.numCacheCursorsFromDiff != 0) {
LOG(log_.info()) << "Loading cache with cursor from num_cursors_from_diff="
@@ -149,6 +157,36 @@ public:
if (loader_ != nullptr)
loader_->wait();
}
private:
bool
loadCacheFromFile()
{
if (not settings_.cacheFileSettings.has_value()) {
return false;
}
LOG(log_.info()) << "Loading ledger cache from " << settings_.cacheFileSettings->path;
auto const minLatestSequence =
backend_->fetchLedgerRange()
.transform([this](data::LedgerRange const& range) {
return std::max(range.maxSequence - settings_.cacheFileSettings->maxAge, range.minSequence);
})
.value_or(0);
auto const [success, duration_ms] = util::timed([&]() {
return cache_.get().loadFromFile(settings_.cacheFileSettings->path, minLatestSequence);
});
if (not success.has_value()) {
LOG(log_.warn()) << "Error loading cache from file: " << success.error();
return false;
}
LOG(log_.info()) << "Loaded cache from file in " << duration_ms
<< " ms. Latest sequence: " << cache_.get().latestLedgerSequence();
backend_->forceUpdateRange(cache_.get().latestLedgerSequence());
return true;
}
};
} // namespace etl

7
src/etl/CacheLoaderSettings.cpp
View File

@@ -26,6 +26,7 @@
#include <cstddef>
#include <cstdint>
#include <string>
#include <utility>
namespace etl {
@@ -63,6 +64,12 @@ makeCacheLoaderSettings(util::config::ClioConfigDefinition const& config)
settings.numCacheMarkers = cache.get<std::size_t>("num_markers");
settings.cachePageFetchSize = cache.get<std::size_t>("page_fetch_size");
if (auto filePath = cache.maybeValue<std::string>("file.path"); filePath.has_value()) {
settings.cacheFileSettings = CacheLoaderSettings::CacheFileSettings{
.path = std::move(filePath).value(), .maxAge = cache.get<uint32_t>("file.max_sequence_age")
};
}
auto const entry = cache.get<std::string>("load");
if (boost::iequals(entry, "sync"))
settings.loadStyle = CacheLoaderSettings::LoadStyle::SYNC;

13
src/etl/CacheLoaderSettings.hpp
View File

@@ -22,6 +22,9 @@
#include "util/config/ConfigDefinition.hpp"
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
namespace etl {
@@ -32,6 +35,15 @@ struct CacheLoaderSettings {
/** @brief Ways to load the cache */
enum class LoadStyle { ASYNC, SYNC, NONE };
/** @brief Settings for cache file operations */
struct CacheFileSettings {
std::string path; /**< path to the file to load cache from on start and save cache to on shutdown */
uint32_t maxAge = 5000; /**< max difference between latest sequence in cache file and DB */
auto
operator<=>(CacheFileSettings const&) const = default;
};
size_t numCacheDiffs = 32; /**< number of diffs to use to generate cursors */
size_t numCacheMarkers = 48; /**< number of markers to use at one time to traverse the ledger */
size_t cachePageFetchSize = 512; /**< number of ledger objects to fetch concurrently per marker */
@@ -40,6 +52,7 @@ struct CacheLoaderSettings {
size_t numCacheCursorsFromAccount = 0; /**< number of cursors to fetch from account_tx */
LoadStyle loadStyle = LoadStyle::ASYNC; /**< how to load the cache */
std::optional<CacheFileSettings> cacheFileSettings; /**< optional settings for cache file operations */
auto
operator<=>(CacheLoaderSettings const&) const = default;

7
src/etl/ETLService.cpp
View File

@@ -213,7 +213,10 @@ ETLService::run()
return;
}
auto const nextSequence = rng->maxSequence + 1;
auto nextSequence = rng->maxSequence + 1;
if (backend_->cache().latestLedgerSequence() != 0) {
nextSequence = backend_->cache().latestLedgerSequence();
}
LOG(log_.debug()) << "Database is populated. Starting monitor loop. sequence = " << nextSequence;
startMonitor(nextSequence);
@@ -333,7 +336,9 @@ ETLService::loadInitialLedgerIfNeeded()
}
LOG(log_.info()) << "Database already populated. Picking up from the tip of history";
if (not backend_->cache().isFull()) {
cacheLoader_->load(rng->maxSequence);
}
return rng;
}

16
src/util/Shasum.cpp
View File

@@ -45,4 +45,20 @@ sha256sumString(std::string_view s)
return ripple::to_string(sha256sum(s));
}
void
Sha256sum::update(void const* data, size_t size)
{
hasher_(data, size);
}
ripple::uint256
Sha256sum::finalize() &&
{
auto const hashData = static_cast<ripple::sha256_hasher::result_type>(hasher_);
ripple::uint256 result;
std::memcpy(result.data(), hashData.data(), hashData.size());
hasher_ = ripple::sha256_hasher{};
return result;
}
} // namespace util

42
src/util/Shasum.hpp
View File

@@ -20,7 +20,9 @@
#pragma once
#include <xrpl/basics/base_uint.h>
#include <xrpl/protocol/digest.h>
#include <cstddef>
#include <string>
#include <string_view>
@@ -43,4 +45,44 @@ sha256sum(std::string_view s);
std::string
sha256sumString(std::string_view s);
/**
* @brief Streaming SHA-256 hasher for large data sets.
*
* This class provides a streaming interface for calculating SHA-256 hashes
* without requiring all data to be in memory at once.
*/
class Sha256sum {
ripple::sha256_hasher hasher_;
public:
/**
* @brief Update hash with data.
*
* @param data Pointer to data to hash.
* @param size Size of data in bytes.
*/
void
update(void const* data, size_t size);
/**
* @brief Update hash with a value.
*
* @param value Value to hash.
*/
template <typename T>
void
update(T const& value)
{
update(&value, sizeof(T));
}
/**
* @brief Finalize hash and return result as ripple::uint256.
*
* @return The SHA-256 hash.
*/
ripple::uint256
finalize() &&;
};
} // namespace util

2
src/util/config/ConfigDefinition.cpp
View File

@@ -359,6 +359,8 @@ getClioConfig()
ConfigValue{ConfigType::Integer}.defaultValue(0).withConstraint(gValidateNumCursors)},
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512).withConstraint(gValidateUint16)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async").withConstraint(gValidateLoadMode)},
{"cache.file.path", ConfigValue{ConfigType::String}.optional()},
{"cache.file.max_sequence_age", ConfigValue{ConfigType::Integer}.defaultValue(5000)},
{"log.channels.[].channel",
Array{ConfigValue{ConfigType::String}.optional().withConstraint(gValidateChannelName)}},

6
src/util/config/ConfigDescription.hpp
View File

@@ -276,6 +276,12 @@ This document provides a list of all available Clio configuration properties in
"If set to `0`, the system defaults to generating cursors based on `cache.num_diffs`."},
KV{.key = "cache.page_fetch_size", .value = "The number of ledger objects to fetch concurrently per marker."},
KV{.key = "cache.load", .value = "The strategy used for Cache loading."},
KV{.key = "cache.file.path",
.value = "The path to a file where cache will be saved to on shutdown and loaded from on startup. "
"If the file couldn't be read Clio will load cache as usual (from DB or from rippled)."},
KV{.key = "cache.file.max_sequence_age",
.value = "Max allowed difference between the latest sequence in DB and in cache file. If the cache file is "
"too old (contains too low latest sequence) Clio will reject using it."},
KV{.key = "log.channels.[].channel", .value = "The name of the log channel."},
KV{.key = "log.channels.[].level", .value = "The log level for the specific log channel."},
KV{.key = "log.level",

9
tests/common/util/MockLedgerCache.hpp
View File

@@ -29,6 +29,7 @@
#include <cstddef>
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
struct MockLedgerCache : data::LedgerCacheInterface {
@@ -77,4 +78,12 @@ struct MockLedgerCache : data::LedgerCacheInterface {
MOCK_METHOD(float, getSuccessorHitRate, (), (const, override));
MOCK_METHOD(void, waitUntilCacheContainsSeq, (uint32_t), (override));
using LoadFromFileReturnType = std::expected<void, std::string>;
MOCK_METHOD(
LoadFromFileReturnType,
loadFromFile,
(std::string const& path, uint32_t minLatestSequence),
(override)
);
};

3
tests/common/util/TmpFile.hpp
View File

@@ -19,10 +19,13 @@
#pragma once
#include <fmt/format.h>
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <ios>
#include <iostream>
#include <string>
#include <string_view>
#include <utility>

3
tests/unit/CMakeLists.txt
View File

@@ -16,6 +16,9 @@ target_sources(
data/cassandra/LedgerHeaderCacheTests.cpp
data/cassandra/RetryPolicyTests.cpp
data/cassandra/SettingsProviderTests.cpp
data/impl/InputFileTests.cpp
data/impl/LedgerCacheFileTests.cpp
data/impl/OutputFileTests.cpp
# Cluster
cluster/ClioNodeTests.cpp
cluster/ClusterCommunicationServiceTests.cpp

177
tests/unit/data/LedgerCacheTests.cpp
View File

@@ -18,11 +18,18 @@
//==============================================================================
#include "data/LedgerCache.hpp"
#include "etl/Models.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TmpFile.hpp"
#include "util/prometheus/Bool.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <string>
#include <vector>
using namespace data;
@@ -65,3 +72,173 @@ TEST_F(LedgerCachePrometheusMetricTest, setFull)
EXPECT_CALL(fullMock, value()).WillOnce(testing::Return(1));
EXPECT_TRUE(cache.isFull());
}
struct LedgerCacheSaveLoadTest : LedgerCacheTest {
ripple::uint256 const key1{1};
ripple::uint256 const key2{2};
std::vector<etl::model::Object> const objs{
etl::model::Object{
.key = key1,
.keyRaw = {},
.data = {1, 2, 3, 4, 5},
.dataRaw = {},
.successor = {},
.predecessor = {},
.type = {}
},
etl::model::Object{
.key = key2,
.keyRaw = {},
.data = {6, 7, 8, 9, 10},
.dataRaw = {},
.successor = {},
.predecessor = {},
.type = {}
}
};
uint32_t const kLEDGER_SEQ = 100;
};
TEST_F(LedgerCacheSaveLoadTest, saveToFileFailsWhenCacheNotFull)
{
auto const tmpFile = TmpFile::empty();
ASSERT_FALSE(cache.isFull());
auto const result = cache.saveToFile(tmpFile.path);
ASSERT_FALSE(result.has_value());
EXPECT_EQ(result.error(), "Ledger cache is not full");
}
TEST_F(LedgerCacheSaveLoadTest, saveAndLoadFromFile)
{
cache.update(objs, kLEDGER_SEQ);
cache.setFull();
ASSERT_TRUE(cache.isFull());
EXPECT_EQ(cache.size(), 2u);
EXPECT_EQ(cache.latestLedgerSequence(), kLEDGER_SEQ);
auto const blob1 = cache.get(key1, kLEDGER_SEQ);
ASSERT_TRUE(blob1.has_value());
EXPECT_EQ(blob1.value(), objs.front().data);
auto const blob2 = cache.get(key2, kLEDGER_SEQ);
ASSERT_TRUE(blob2.has_value());
EXPECT_EQ(blob2.value(), objs.back().data);
auto const tmpFile = TmpFile::empty();
auto const saveResult = cache.saveToFile(tmpFile.path);
ASSERT_TRUE(saveResult.has_value()) << "Save failed: " << saveResult.error();
LedgerCache newCache;
auto const loadResult = newCache.loadFromFile(tmpFile.path, 0);
ASSERT_TRUE(loadResult.has_value()) << "Load failed: " << loadResult.error();
EXPECT_TRUE(newCache.isFull());
EXPECT_EQ(newCache.size(), 2u);
EXPECT_EQ(newCache.latestLedgerSequence(), kLEDGER_SEQ);
auto const loadedBlob1 = newCache.get(key1, kLEDGER_SEQ);
ASSERT_TRUE(loadedBlob1.has_value());
EXPECT_EQ(loadedBlob1.value(), blob1);
auto const loadedBlob2 = newCache.get(key2, kLEDGER_SEQ);
ASSERT_TRUE(loadedBlob2.has_value());
EXPECT_EQ(loadedBlob2.value(), blob2);
EXPECT_EQ(newCache.latestLedgerSequence(), cache.latestLedgerSequence());
}
TEST_F(LedgerCacheSaveLoadTest, saveAndLoadFromFileWithDeletedObjects)
{
cache.update(objs, kLEDGER_SEQ - 1);
auto objsCopy = objs;
objsCopy.front().data = {};
cache.update(objsCopy, kLEDGER_SEQ);
cache.setFull();
// Verify deleted object is accessible via getDeleted
auto const blob1 = cache.get(key1, kLEDGER_SEQ);
ASSERT_FALSE(blob1.has_value());
auto const blob2 = cache.get(key2, kLEDGER_SEQ);
ASSERT_TRUE(blob2.has_value());
EXPECT_EQ(blob2.value(), objs.back().data);
auto const deletedBlob = cache.getDeleted(key1, kLEDGER_SEQ - 1);
ASSERT_TRUE(deletedBlob.has_value());
EXPECT_EQ(deletedBlob.value(), objs.front().data);
// Save and load
auto const tmpFile = TmpFile::empty();
auto saveResult = cache.saveToFile(tmpFile.path);
ASSERT_TRUE(saveResult.has_value()) << "Save failed: " << saveResult.error();
LedgerCache newCache;
auto loadResult = newCache.loadFromFile(tmpFile.path, 0);
ASSERT_TRUE(loadResult.has_value()) << "Load failed: " << loadResult.error();
// Verify deleted object is preserved
auto const loadedDeletedBlob = newCache.getDeleted(key1, kLEDGER_SEQ - 1);
ASSERT_TRUE(loadedDeletedBlob.has_value());
EXPECT_EQ(loadedDeletedBlob.value(), deletedBlob);
// Verify active object
auto const loadedBlob1 = newCache.get(key1, kLEDGER_SEQ);
ASSERT_FALSE(loadedBlob1.has_value());
auto const loadedBlob2 = newCache.get(key2, kLEDGER_SEQ);
ASSERT_TRUE(loadedBlob2.has_value());
EXPECT_EQ(loadedBlob2.value(), blob2);
EXPECT_TRUE(newCache.isFull());
EXPECT_EQ(newCache.latestLedgerSequence(), cache.latestLedgerSequence());
}
TEST_F(LedgerCacheTest, SaveFailedDueToFilePermissions)
{
cache.setFull();
auto const result = cache.saveToFile("/");
ASSERT_FALSE(result.has_value());
EXPECT_FALSE(result.error().empty());
}
TEST_F(LedgerCacheTest, loadFromNonExistentFileReturnsError)
{
auto const result = cache.loadFromFile("/nonexistent/path/cache.dat", 0);
ASSERT_FALSE(result.has_value());
EXPECT_FALSE(result.error().empty());
}
TEST_F(LedgerCacheSaveLoadTest, RejectOldCacheFile)
{
uint32_t const cacheSeq = 100;
cache.update(objs, cacheSeq);
cache.setFull();
auto const tmpFile = TmpFile::empty();
auto const saveResult = cache.saveToFile(tmpFile.path);
ASSERT_TRUE(saveResult.has_value());
LedgerCache newCache;
auto const loadResult = newCache.loadFromFile(tmpFile.path, cacheSeq + 1);
EXPECT_FALSE(loadResult.has_value());
EXPECT_THAT(loadResult.error(), ::testing::HasSubstr("too low"));
}
TEST_F(LedgerCacheSaveLoadTest, AcceptRecentCacheFile)
{
uint32_t const cacheSeq = 100;
cache.update(objs, cacheSeq);
cache.setFull();
auto const tmpFile = TmpFile::empty();
auto const saveResult = cache.saveToFile(tmpFile.path);
ASSERT_TRUE(saveResult.has_value());
LedgerCache newCache;
auto const loadResult = newCache.loadFromFile(tmpFile.path, cacheSeq - 1);
ASSERT_TRUE(loadResult.has_value());
EXPECT_EQ(newCache.latestLedgerSequence(), cacheSeq);
}

196
tests/unit/data/impl/InputFileTests.cpp Normal file
View File

@@ -0,0 +1,196 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/impl/InputFile.hpp"
#include "util/Shasum.hpp"
#include "util/TmpFile.hpp"
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <string>
#include <vector>
using namespace data::impl;
struct InputFileTest : ::testing::Test {};
TEST_F(InputFileTest, ConstructorWithValidFile)
{
auto const tmpFile = TmpFile{"Hello, World!"};
InputFile inputFile(tmpFile.path);
EXPECT_TRUE(inputFile.isOpen());
}
TEST_F(InputFileTest, ConstructorWithInvalidFile)
{
InputFile inputFile("/nonexistent/path/file.txt");
EXPECT_FALSE(inputFile.isOpen());
char i = 0;
EXPECT_FALSE(inputFile.read(i));
EXPECT_FALSE(inputFile.readRaw(&i, 1));
}
TEST_F(InputFileTest, ReadRawFromFile)
{
std::string const content = "Test content for reading";
auto tmpFile = TmpFile{content};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::vector<char> buffer(content.size());
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
EXPECT_EQ(std::string(buffer.data(), buffer.size()), content);
}
TEST_F(InputFileTest, ReadRawFromFilePartial)
{
std::string content = "Hello, World!";
auto tmpFile = TmpFile{content};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::vector<char> buffer(3);
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
EXPECT_EQ(std::string(buffer.data(), buffer.size()), "Hel"); // codespell:ignore
buffer.resize(6);
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
EXPECT_EQ(std::string(buffer.data(), buffer.size()), "lo, Wo");
buffer.resize(4);
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
EXPECT_EQ(std::string(buffer.data(), buffer.size()), "rld!");
}
TEST_F(InputFileTest, ReadRawAfterEnd)
{
std::string content = "Test";
auto tmpFile = TmpFile{content};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::vector<char> buffer(content.size());
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
char extraByte = 0;
EXPECT_FALSE(inputFile.readRaw(&extraByte, 1));
}
TEST_F(InputFileTest, ReadRawFromFileExceedsSize)
{
auto tmpFile = TmpFile{"Test"};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::vector<char> buffer(10); // Larger than file content
EXPECT_FALSE(inputFile.readRaw(buffer.data(), buffer.size()));
}
TEST_F(InputFileTest, ReadTemplateMethod)
{
auto tmpFile = TmpFile{"\x01\x02\x03\x04"};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::uint32_t value{0};
bool success = inputFile.read(value);
EXPECT_TRUE(success);
// Note: The actual value depends on endianness
EXPECT_NE(value, 0u);
}
TEST_F(InputFileTest, ReadTemplateMethodFailure)
{
auto tmpFile = TmpFile{"Hi"}; // Only 2 bytes
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
std::uint64_t value{0}; // Trying to read 8 bytes
EXPECT_FALSE(inputFile.read(value));
}
TEST_F(InputFileTest, ReadFromEmptyFile)
{
auto tmpFile = TmpFile::empty();
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
char byte = 0;
EXPECT_FALSE(inputFile.readRaw(&byte, 1));
}
TEST_F(InputFileTest, HashOfEmptyFile)
{
auto tmpFile = TmpFile::empty();
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
EXPECT_EQ(inputFile.hash(), util::sha256sum(""));
}
TEST_F(InputFileTest, HashAfterReading)
{
std::string const content = "Hello, World!";
auto tmpFile = TmpFile{content};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
EXPECT_EQ(inputFile.hash(), util::sha256sum(""));
std::vector<char> buffer(content.size());
EXPECT_TRUE(inputFile.readRaw(buffer.data(), buffer.size()));
EXPECT_EQ(std::string(buffer.data(), buffer.size()), content);
EXPECT_EQ(inputFile.hash(), util::sha256sum(content));
}
TEST_F(InputFileTest, HashProgressesWithReading)
{
std::string const content = "Hello, World!";
auto tmpFile = TmpFile{content};
InputFile inputFile(tmpFile.path);
ASSERT_TRUE(inputFile.isOpen());
EXPECT_EQ(inputFile.hash(), util::sha256sum(""));
// Read first part
std::vector<char> buffer1(5);
EXPECT_TRUE(inputFile.readRaw(buffer1.data(), buffer1.size()));
EXPECT_EQ(inputFile.hash(), util::sha256sum("Hello"));
// Read second part
std::vector<char> buffer2(8);
EXPECT_TRUE(inputFile.readRaw(buffer2.data(), buffer2.size()));
EXPECT_EQ(inputFile.hash(), util::sha256sum(content));
}

723
tests/unit/data/impl/LedgerCacheFileTests.cpp Normal file
View File

@@ -0,0 +1,723 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/LedgerCache.hpp"
#include "data/impl/LedgerCacheFile.hpp"
#include "util/NameGenerator.hpp"
#include "util/TmpFile.hpp"
#include <fmt/format.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <ios>
#include <limits>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
using namespace data::impl;
struct LedgerCacheFileTestBase : ::testing::Test {
struct DataSizeParams {
size_t mapEntries;
size_t deletedEntries;
size_t blobSize;
std::string description;
};
enum class CorruptionType {
InvalidVersion,
CorruptedSeparator1, // After header
CorruptedSeparator2, // After map hash
CorruptedSeparator3, // After deleted hash
MapKeyCorrupted,
MapSeqCorrupted,
MapBlobSizeCorrupted,
MapBlobDataCorrupted,
DeletedKeyCorrupted,
DeletedSeqCorrupted,
DeletedBlobSizeCorrupted,
DeletedBlobDataCorrupted,
HeaderLatestSeqCorrupted
};
struct CorruptionParams {
CorruptionType type;
std::string description;
};
struct EntryOffsets {
size_t keyOffset;
size_t seqOffset;
size_t blobSizeOffset;
size_t blobDataOffset;
};
struct FileOffsets {
size_t headerOffset;
size_t separator1Offset;
size_t mapStartOffset;
std::vector<EntryOffsets> mapEntries;
size_t separator2Offset;
size_t deletedStartOffset;
std::vector<EntryOffsets> deletedEntries;
size_t separator3Offset;
size_t hashOffset;
static FileOffsets
calculate(LedgerCacheFile::DataView const& dataView)
{
FileOffsets offsets{};
size_t currentOffset = 0;
offsets.headerOffset = currentOffset;
currentOffset += sizeof(LedgerCacheFile::Header);
offsets.separator1Offset = currentOffset;
currentOffset += 16;
// Map entries
offsets.mapStartOffset = currentOffset;
for (auto const& [key, entry] : dataView.map) {
EntryOffsets entryOffsets{};
entryOffsets.keyOffset = currentOffset;
entryOffsets.seqOffset = currentOffset + 32; // uint256 size
entryOffsets.blobSizeOffset = currentOffset + 32 + 4; // + uint32 size
entryOffsets.blobDataOffset = currentOffset + 32 + 4 + 8; // + size_t size
offsets.mapEntries.push_back(entryOffsets);
currentOffset += 32 + 4 + 8 + entry.blob.size(); // key + seq + size + blob
}
// Separator 2 (after map entries)
offsets.separator2Offset = currentOffset;
currentOffset += 16;
// Deleted entries
offsets.deletedStartOffset = currentOffset;
for (auto const& [key, entry] : dataView.deleted) {
EntryOffsets entryOffsets{};
entryOffsets.keyOffset = currentOffset;
entryOffsets.seqOffset = currentOffset + 32;
entryOffsets.blobSizeOffset = currentOffset + 32 + 4;
entryOffsets.blobDataOffset = currentOffset + 32 + 4 + 8;
offsets.deletedEntries.push_back(entryOffsets);
currentOffset += 32 + 4 + 8 + entry.blob.size();
}
// Separator 3 (after deleted entries)
offsets.separator3Offset = currentOffset;
currentOffset += 16;
// Overall file hash
offsets.hashOffset = currentOffset;
return offsets;
}
};
~LedgerCacheFileTestBase() override
{
auto const pathWithNewPrefix = fmt::format("{}.new", tmpFile.path);
if (std::filesystem::exists(pathWithNewPrefix))
std::filesystem::remove(pathWithNewPrefix);
}
static std::vector<DataSizeParams> const kDATA_SIZE_PARAMS;
static std::vector<CorruptionParams> const kCORRUPTION_PARAMS;
TmpFile tmpFile = TmpFile::empty();
static uint32_t constexpr kLATEST_SEQUENCE = 12345;
static LedgerCacheFile::Data
createTestData(size_t mapSize, size_t deletedSize, size_t blobSize)
{
LedgerCacheFile::Data data;
data.latestSeq = kLATEST_SEQUENCE;
for (size_t i = 0; i < mapSize; ++i) {
ripple::uint256 key;
std::memset(key.data(), static_cast<int>(i), ripple::uint256::size());
data::LedgerCache::CacheEntry entry;
entry.seq = static_cast<uint32_t>(1000 + i);
entry.blob.resize(blobSize);
std::memset(entry.blob.data(), static_cast<int>(i + 100), blobSize);
data.map.emplace(key, std::move(entry));
}
for (size_t i = 0; i < deletedSize; ++i) {
ripple::uint256 key;
std::memset(key.data(), static_cast<int>(i + 200), ripple::uint256::size());
data::LedgerCache::CacheEntry entry;
entry.seq = static_cast<uint32_t>(2000 + i);
entry.blob.resize(blobSize);
std::memset(entry.blob.data(), static_cast<int>(i + 250), blobSize);
data.deleted.emplace(key, std::move(entry));
}
return data;
}
static LedgerCacheFile::DataView
toDataView(LedgerCacheFile::Data const& data)
{
return LedgerCacheFile::DataView{.latestSeq = data.latestSeq, .map = data.map, .deleted = data.deleted};
}
void
corruptFile(CorruptionType type, LedgerCacheFile::DataView const& dataView) const
{
std::fstream file(tmpFile.path, std::ios::in | std::ios::out | std::ios::binary);
ASSERT_TRUE(file.is_open());
auto const offsets = FileOffsets::calculate(dataView);
switch (type) {
case CorruptionType::InvalidVersion:
file.seekp(offsets.headerOffset);
{
uint32_t invalidVersion = 999;
file.write(reinterpret_cast<char const*>(&invalidVersion), sizeof(invalidVersion));
}
break;
case CorruptionType::CorruptedSeparator1:
file.seekp(offsets.separator1Offset);
{
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::CorruptedSeparator2:
file.seekp(offsets.separator2Offset);
{
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::CorruptedSeparator3:
file.seekp(offsets.separator3Offset);
{
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::MapKeyCorrupted:
if (!offsets.mapEntries.empty()) {
file.seekp(offsets.mapEntries[0].keyOffset);
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::MapSeqCorrupted:
if (!offsets.mapEntries.empty()) {
file.seekp(offsets.mapEntries[0].seqOffset);
uint32_t const corruptSeq = std::numeric_limits<uint32_t>::max();
file.write(reinterpret_cast<char const*>(&corruptSeq), sizeof(corruptSeq));
}
break;
case CorruptionType::MapBlobSizeCorrupted:
if (!offsets.mapEntries.empty()) {
file.seekp(offsets.mapEntries[0].blobSizeOffset);
size_t const corruptSize = std::numeric_limits<size_t>::max();
file.write(reinterpret_cast<char const*>(&corruptSize), sizeof(corruptSize));
}
break;
case CorruptionType::MapBlobDataCorrupted:
if (!offsets.mapEntries.empty() && !dataView.map.begin()->second.blob.empty()) {
file.seekp(offsets.mapEntries[0].blobDataOffset);
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::DeletedKeyCorrupted:
if (!offsets.deletedEntries.empty()) {
file.seekp(offsets.deletedEntries[0].keyOffset);
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::DeletedSeqCorrupted:
if (!offsets.deletedEntries.empty()) {
file.seekp(offsets.deletedEntries[0].seqOffset);
uint32_t const corruptSeq = std::numeric_limits<uint32_t>::max();
file.write(reinterpret_cast<char const*>(&corruptSeq), sizeof(corruptSeq));
}
break;
case CorruptionType::DeletedBlobSizeCorrupted:
if (!offsets.deletedEntries.empty()) {
file.seekp(offsets.deletedEntries[0].blobSizeOffset);
size_t const corruptSize = std::numeric_limits<size_t>::max();
file.write(reinterpret_cast<char const*>(&corruptSize), sizeof(corruptSize));
}
break;
case CorruptionType::DeletedBlobDataCorrupted:
if (!offsets.deletedEntries.empty() && !dataView.deleted.begin()->second.blob.empty()) {
file.seekp(offsets.deletedEntries[0].blobDataOffset);
char corruptByte = static_cast<char>(0xFF);
file.write(&corruptByte, 1);
}
break;
case CorruptionType::HeaderLatestSeqCorrupted:
file.seekp(offsets.headerOffset + sizeof(uint32_t)); // skip version
{
uint32_t corruptSeq = 0; // set to 0 to fail validation if minLatestSequence > 0
file.write(reinterpret_cast<char const*>(&corruptSeq), sizeof(corruptSeq));
}
break;
}
}
static void
verifyDataEquals(LedgerCacheFile::Data const& expected, LedgerCacheFile::Data const& actual)
{
EXPECT_EQ(expected.latestSeq, actual.latestSeq);
EXPECT_EQ(expected.map.size(), actual.map.size());
EXPECT_EQ(expected.deleted.size(), actual.deleted.size());
for (auto const& [key, entry] : expected.map) {
auto it = actual.map.find(key);
ASSERT_NE(it, actual.map.end()) << "Key not found in actual map";
EXPECT_EQ(entry.seq, it->second.seq);
EXPECT_EQ(entry.blob, it->second.blob);
}
for (auto const& [key, entry] : expected.deleted) {
auto it = actual.deleted.find(key);
ASSERT_NE(it, actual.deleted.end()) << "Key not found in actual deleted";
EXPECT_EQ(entry.seq, it->second.seq);
EXPECT_EQ(entry.blob, it->second.blob);
}
}
};
std::vector<LedgerCacheFileTestBase::DataSizeParams> const LedgerCacheFileTestBase::kDATA_SIZE_PARAMS = {
{.mapEntries = 0, .deletedEntries = 0, .blobSize = 0, .description = "empty"},
{.mapEntries = 1, .deletedEntries = 0, .blobSize = 10, .description = "single_map_small_blob"},
{.mapEntries = 0, .deletedEntries = 1, .blobSize = 100, .description = "single_deleted_medium_blob"},
{.mapEntries = 5, .deletedEntries = 3, .blobSize = 1000, .description = "multiple_entries_large_blob"},
{.mapEntries = 10, .deletedEntries = 10, .blobSize = 50000, .description = "many_entries_huge_blob"}
};
std::vector<LedgerCacheFileTestBase::CorruptionParams> const LedgerCacheFileTestBase::kCORRUPTION_PARAMS = {
{.type = CorruptionType::InvalidVersion, .description = "invalid_version"},
{.type = CorruptionType::CorruptedSeparator1, .description = "corrupted_separator1"},
{.type = CorruptionType::CorruptedSeparator2, .description = "corrupted_separator2"},
{.type = CorruptionType::CorruptedSeparator3, .description = "corrupted_separator3"},
{.type = CorruptionType::MapKeyCorrupted, .description = "map_key_corrupted"},
{.type = CorruptionType::MapSeqCorrupted, .description = "map_seq_corrupted"},
{.type = CorruptionType::MapBlobSizeCorrupted, .description = "map_blob_size_corrupted"},
{.type = CorruptionType::MapBlobDataCorrupted, .description = "map_blob_data_corrupted"},
{.type = CorruptionType::DeletedKeyCorrupted, .description = "deleted_key_corrupted"},
{.type = CorruptionType::DeletedSeqCorrupted, .description = "deleted_seq_corrupted"},
{.type = CorruptionType::DeletedBlobSizeCorrupted, .description = "deleted_blob_size_corrupted"},
{.type = CorruptionType::DeletedBlobDataCorrupted, .description = "deleted_blob_data_corrupted"},
{.type = CorruptionType::HeaderLatestSeqCorrupted, .description = "header_latest_seq_corrupted"}
};
struct LedgerCacheFileTest : LedgerCacheFileTestBase,
::testing::WithParamInterface<LedgerCacheFileTestBase::DataSizeParams> {
static std::string
roundTripParamName(::testing::TestParamInfo<DataSizeParams> const& info)
{
return info.param.description;
}
};
INSTANTIATE_TEST_SUITE_P(
AllDataSizes,
LedgerCacheFileTest,
::testing::ValuesIn(LedgerCacheFileTestBase::kDATA_SIZE_PARAMS),
LedgerCacheFileTest::roundTripParamName
);
TEST_P(LedgerCacheFileTest, WriteAndReadData)
{
auto dataParams = GetParam();
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(dataParams.mapEntries, dataParams.deletedEntries, dataParams.blobSize);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value()) << "Failed to write: " << writeResult.error();
EXPECT_TRUE(std::filesystem::exists(tmpFile.path));
EXPECT_GT(std::filesystem::file_size(tmpFile.path), 0u);
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value()) << "Failed to read: " << readResult.error();
verifyDataEquals(testData, readResult.value());
}
struct LedgerCacheFileCorruptionTest : LedgerCacheFileTestBase,
::testing::WithParamInterface<LedgerCacheFileTestBase::CorruptionParams> {
static std::string
corruptionParamName(::testing::TestParamInfo<CorruptionParams> const& info)
{
return info.param.description;
}
};
INSTANTIATE_TEST_SUITE_P(
AllCorruptions,
LedgerCacheFileCorruptionTest,
::testing::ValuesIn(LedgerCacheFileTestBase::kCORRUPTION_PARAMS),
LedgerCacheFileCorruptionTest::corruptionParamName
);
TEST_P(LedgerCacheFileCorruptionTest, HandleCorruption)
{
auto corruptionParams = GetParam();
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(3, 2, 100);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value()) << "Failed to write: " << writeResult.error();
corruptFile(corruptionParams.type, dataView);
auto readResult = cacheFile.read(0);
EXPECT_FALSE(readResult.has_value()) << "Should have failed to read corrupted file";
std::string const& error = readResult.error();
switch (corruptionParams.type) {
case CorruptionType::InvalidVersion:
EXPECT_THAT(error, ::testing::HasSubstr("wrong version"));
break;
case CorruptionType::CorruptedSeparator1:
case CorruptionType::CorruptedSeparator2:
case CorruptionType::CorruptedSeparator3:
EXPECT_THAT(error, ::testing::HasSubstr("Separator verification failed"));
break;
case CorruptionType::MapKeyCorrupted:
case CorruptionType::MapSeqCorrupted:
EXPECT_FALSE(error.empty());
break;
case CorruptionType::MapBlobSizeCorrupted:
EXPECT_THAT(
error,
::testing::AnyOf(
::testing::HasSubstr("Error reading cache file"),
::testing::HasSubstr("Failed to read blob"),
::testing::HasSubstr("Hash file corruption detected")
)
);
break;
case CorruptionType::MapBlobDataCorrupted:
EXPECT_THAT(
error,
::testing::AnyOf(
::testing::HasSubstr("Hash file corruption detected"),
::testing::HasSubstr("Error reading cache file")
)
);
break;
case CorruptionType::DeletedKeyCorrupted:
case CorruptionType::DeletedSeqCorrupted:
EXPECT_FALSE(error.empty());
break;
case CorruptionType::DeletedBlobSizeCorrupted:
EXPECT_THAT(
error,
::testing::AnyOf(
::testing::HasSubstr("Error reading cache file"),
::testing::HasSubstr("Failed to read blob"),
::testing::HasSubstr("Hash file corruption detected")
)
);
break;
case CorruptionType::DeletedBlobDataCorrupted:
EXPECT_THAT(
error,
::testing::AnyOf(
::testing::HasSubstr("Hash file corruption detected"),
::testing::HasSubstr("Error reading cache file")
)
);
break;
case CorruptionType::HeaderLatestSeqCorrupted:
EXPECT_THAT(error, ::testing::HasSubstr("Hash file corruption detected"));
break;
}
}
struct LedgerCacheFileEdgeCaseTest : LedgerCacheFileTestBase {};
TEST_F(LedgerCacheFileEdgeCaseTest, NonExistingFile)
{
LedgerCacheFile invalidPathFile("/invalid/path/file.cache");
auto testData = createTestData(1, 1, 10);
auto dataView = toDataView(testData);
auto writeResult = invalidPathFile.write(dataView);
EXPECT_FALSE(writeResult.has_value());
EXPECT_THAT(writeResult.error(), ::testing::HasSubstr("Couldn't open file"));
auto readResult = invalidPathFile.read(0);
EXPECT_FALSE(readResult.has_value());
EXPECT_THAT(readResult.error(), ::testing::HasSubstr("Couldn't open file"));
}
TEST_F(LedgerCacheFileEdgeCaseTest, MaxSequenceNumber)
{
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(1, 1, 10);
testData.latestSeq = std::numeric_limits<uint32_t>::max();
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, ZeroSizedBlobs)
{
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(3, 2, 0);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, SpecialKeyPatterns)
{
LedgerCacheFile cacheFile(tmpFile.path);
LedgerCacheFile::Data testData;
testData.latestSeq = 100;
ripple::uint256 zeroKey;
std::memset(zeroKey.data(), 0, ripple::uint256::size());
testData.map.emplace(zeroKey, data::LedgerCache::CacheEntry{.seq = 1, .blob = {1, 2, 3}});
ripple::uint256 onesKey;
std::memset(onesKey.data(), 0xFF, ripple::uint256::size());
testData.map.emplace(onesKey, data::LedgerCache::CacheEntry{.seq = 2, .blob = {4, 5, 6}});
ripple::uint256 altKey;
for (size_t i = 0; i < ripple::uint256::size(); ++i) {
altKey.data()[i] = static_cast<unsigned char>(((i % 2) != 0u) ? 0xAA : 0x55);
}
testData.deleted.emplace(altKey, data::LedgerCache::CacheEntry{.seq = 3, .blob = {7, 8, 9}});
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, LargeBlobs)
{
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(1, 1, 1024 * 1024);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, SequenceNumber)
{
LedgerCacheFile cacheFile(tmpFile.path);
LedgerCacheFile::Data testData;
testData.latestSeq = 0;
ripple::uint256 key1, key2, key3;
std::memset(key1.data(), 1, ripple::uint256::size());
std::memset(key2.data(), 2, ripple::uint256::size());
std::memset(key3.data(), 3, ripple::uint256::size());
testData.map.emplace(key1, data::LedgerCache::CacheEntry{.seq = 0, .blob = {1}});
testData.map.emplace(key2, data::LedgerCache::CacheEntry{.seq = std::numeric_limits<uint32_t>::max(), .blob = {2}});
testData.deleted.emplace(
key3, data::LedgerCache::CacheEntry{.seq = std::numeric_limits<uint32_t>::max() / 2, .blob = {3}}
);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, OnlyMapEntries)
{
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(5, 0, 100);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, OnlyDeletedEntries)
{
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(0, 5, 100);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(0);
ASSERT_TRUE(readResult.has_value());
verifyDataEquals(testData, readResult.value());
}
TEST_F(LedgerCacheFileEdgeCaseTest, WriteCreatesFileWithSuffixNew)
{
// The test causes failure of rename operation by creating destination as directory
std::filesystem::remove(tmpFile.path);
std::filesystem::create_directory(tmpFile.path);
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(1, 1, 10);
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
EXPECT_FALSE(writeResult.has_value());
auto newFilePath = fmt::format("{}.new", tmpFile.path);
EXPECT_THAT(writeResult.error(), ::testing::HasSubstr(newFilePath));
EXPECT_TRUE(std::filesystem::exists(newFilePath));
EXPECT_TRUE(std::filesystem::is_regular_file(newFilePath));
}
struct LedgerCacheFileMinSequenceValidationParams {
uint32_t latestSeq;
uint32_t minLatestSeq;
bool shouldSucceed;
std::string testName;
};
struct LedgerCacheFileMinSequenceValidationTest
: LedgerCacheFileTestBase,
::testing::WithParamInterface<LedgerCacheFileMinSequenceValidationParams> {};
INSTANTIATE_TEST_SUITE_P(
LedgerCacheFileMinSequenceValidationTests,
LedgerCacheFileMinSequenceValidationTest,
::testing::Values(
LedgerCacheFileMinSequenceValidationParams{
.latestSeq = 1000u,
.minLatestSeq = 500u,
.shouldSucceed = true,
.testName = "accept_when_min_less_than_latest"
},
LedgerCacheFileMinSequenceValidationParams{
.latestSeq = 1000u,
.minLatestSeq = 2000u,
.shouldSucceed = false,
.testName = "reject_when_min_greater_than_latest"
},
LedgerCacheFileMinSequenceValidationParams{
.latestSeq = 1000u,
.minLatestSeq = 1000u,
.shouldSucceed = true,
.testName = "accept_when_min_equals_latest"
},
LedgerCacheFileMinSequenceValidationParams{
.latestSeq = 0u,
.minLatestSeq = 0u,
.shouldSucceed = true,
.testName = "accept_zero_sequence"
}
),
tests::util::kNAME_GENERATOR
);
TEST_P(LedgerCacheFileMinSequenceValidationTest, ValidateMinSequence)
{
auto const params = GetParam();
auto const latestSeq = params.latestSeq;
auto const minLatestSeq = params.minLatestSeq;
auto const shouldSucceed = params.shouldSucceed;
LedgerCacheFile cacheFile(tmpFile.path);
auto testData = createTestData(3, 2, 100);
testData.latestSeq = latestSeq;
auto dataView = toDataView(testData);
auto writeResult = cacheFile.write(dataView);
ASSERT_TRUE(writeResult.has_value());
auto readResult = cacheFile.read(minLatestSeq);
if (shouldSucceed) {
ASSERT_TRUE(readResult.has_value()) << "Expected read to succeed but got error: " << readResult.error();
EXPECT_EQ(readResult.value().latestSeq, latestSeq);
} else {
EXPECT_FALSE(readResult.has_value()) << "Expected read to fail but it succeeded";
EXPECT_THAT(readResult.error(), ::testing::HasSubstr("too low"));
}
}

169
tests/unit/data/impl/OutputFileTests.cpp Normal file
View File

@@ -0,0 +1,169 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/impl/OutputFile.hpp"
#include "util/Shasum.hpp"
#include "util/TmpFile.hpp"
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <cstddef>
#include <cstdint>
#include <fstream>
#include <ios>
#include <iterator>
#include <numbers>
#include <string>
#include <vector>
using namespace data::impl;
struct OutputFileTest : ::testing::Test {
TmpFile tmpFile = TmpFile::empty();
std::string
readFileContents() const
{
std::ifstream ifs(tmpFile.path, std::ios::binary);
return std::string{std::istreambuf_iterator<char>{ifs}, std::istreambuf_iterator<char>{}};
}
};
TEST_F(OutputFileTest, ConstructorOpensFile)
{
OutputFile file(tmpFile.path);
EXPECT_TRUE(file.isOpen());
}
TEST_F(OutputFileTest, NonExistingFile)
{
std::string const invalidPath = "/invalid/nonexistent/directory/file.dat";
OutputFile file(invalidPath);
EXPECT_FALSE(file.isOpen());
}
TEST_F(OutputFileTest, WriteBasicTypes)
{
uint32_t const intValue = 0x12345678;
double const doubleValue = std::numbers::pi;
char const charValue = 'A';
{
OutputFile file(tmpFile.path);
file.write(intValue);
file.write(doubleValue);
file.write(charValue);
}
std::string contents = readFileContents();
EXPECT_EQ(contents.size(), sizeof(intValue) + sizeof(doubleValue) + sizeof(charValue));
auto* data = reinterpret_cast<char const*>(contents.data());
EXPECT_EQ(*reinterpret_cast<uint32_t const*>(data), intValue);
EXPECT_EQ(*reinterpret_cast<double const*>(data + sizeof(intValue)), doubleValue);
EXPECT_EQ(*(data + sizeof(intValue) + sizeof(doubleValue)), charValue);
}
TEST_F(OutputFileTest, WriteArray)
{
std::vector<uint32_t> const data = {0x11111111, 0x22222222, 0x33333333, 0x44444444};
{
OutputFile file(tmpFile.path);
file.write(data.data(), data.size() * sizeof(uint32_t));
}
std::string contents = readFileContents();
EXPECT_EQ(contents.size(), data.size() * sizeof(uint32_t));
auto* readData = reinterpret_cast<uint32_t const*>(contents.data());
for (size_t i = 0; i < data.size(); ++i) {
EXPECT_EQ(readData[i], data[i]);
}
}
TEST_F(OutputFileTest, WriteRawData)
{
std::string const testData = "Hello, World!";
{
OutputFile file(tmpFile.path);
file.writeRaw(testData.data(), testData.size());
}
std::string contents = readFileContents();
EXPECT_EQ(contents, testData);
}
TEST_F(OutputFileTest, WriteMultipleChunks)
{
std::string chunk1 = "First chunk";
std::string chunk2 = "Second chunk";
std::string chunk3 = "Third chunk";
{
OutputFile file(tmpFile.path);
file.writeRaw(chunk1.data(), chunk1.size());
file.writeRaw(chunk2.data(), chunk2.size());
file.writeRaw(chunk3.data(), chunk3.size());
}
std::string contents = readFileContents();
EXPECT_EQ(contents, chunk1 + chunk2 + chunk3);
}
TEST_F(OutputFileTest, HashOfEmptyFile)
{
OutputFile file(tmpFile.path);
ASSERT_TRUE(file.isOpen());
// Hash of empty file should match SHA256 of empty string
EXPECT_EQ(file.hash(), util::sha256sum(""));
}
TEST_F(OutputFileTest, HashAfterWriting)
{
std::string const testData = "Hello, World!";
{
OutputFile file(tmpFile.path);
file.writeRaw(testData.data(), testData.size());
// Hash should match SHA256 of the written data
EXPECT_EQ(file.hash(), util::sha256sum(testData));
}
}
TEST_F(OutputFileTest, HashProgressesWithWrites)
{
std::string const part1 = "Hello, ";
std::string const part2 = "World!";
std::string const combined = part1 + part2;
OutputFile file(tmpFile.path);
ASSERT_TRUE(file.isOpen());
EXPECT_EQ(file.hash(), util::sha256sum(""));
file.writeRaw(part1.data(), part1.size());
EXPECT_EQ(file.hash(), util::sha256sum(part1));
file.writeRaw(part2.data(), part2.size());
EXPECT_EQ(file.hash(), util::sha256sum(combined));
}

37
tests/unit/etl/CacheLoaderSettingsTests.cpp
View File

@@ -25,6 +25,8 @@
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp>
#include <fmt/core.h>
#include <fmt/format.h>
#include <gtest/gtest.h>
namespace json = boost::json;
@@ -42,7 +44,9 @@ generateDefaultCacheConfig()
{"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0)},
{"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0)},
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async")}}
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async")},
{"cache.file.path", ConfigValue{ConfigType::String}.optional()},
{"cache.file.max_sequence_age", ConfigValue{ConfigType::Integer}.defaultValue(5000)}}
};
}
@@ -134,3 +138,34 @@ TEST_F(CacheLoaderSettingsTest, NoLoadStyleCorrectlyPropagatedThroughConfig)
EXPECT_TRUE(settings.isDisabled());
}
}
TEST_F(CacheLoaderSettingsTest, CacheFilePathCorrectlyPropagatedThroughConfig)
{
static constexpr auto kCACHE_FILE_PATH = "/path/to/cache.dat";
auto const jsonStr = fmt::format(R"JSON({{"cache": {{"file": {{"path": "{}"}}}}}})JSON", kCACHE_FILE_PATH);
auto const cfg = getParseCacheConfig(json::parse(jsonStr));
auto const settings = makeCacheLoaderSettings(cfg);
ASSERT_TRUE(settings.cacheFileSettings.has_value());
EXPECT_EQ(settings.cacheFileSettings->path, kCACHE_FILE_PATH);
}
TEST_F(CacheLoaderSettingsTest, CacheFilePathNotSetWhenAbsentFromConfig)
{
auto const cfg = generateDefaultCacheConfig();
auto const settings = makeCacheLoaderSettings(cfg);
EXPECT_FALSE(settings.cacheFileSettings.has_value());
}
TEST_F(CacheLoaderSettingsTest, MaxSequenceLagPropagatedThoughConfig)
{
auto const seq = 1234;
auto const jsonStr =
fmt::format(R"JSON({{"cache": {{"file": {{"path": "doesnt_matter", "max_sequence_age": {} }}}}}})JSON", seq);
auto const cfg = getParseCacheConfig(json::parse(jsonStr));
auto const settings = makeCacheLoaderSettings(cfg);
ASSERT_TRUE(settings.cacheFileSettings.has_value());
EXPECT_EQ(settings.cacheFileSettings->maxAge, seq);
}

203
tests/unit/etl/CacheLoaderTests.cpp
View File

@@ -33,9 +33,13 @@
#include <boost/json/parse.hpp>
#include <boost/json/value.hpp>
#include <fmt/format.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
namespace json = boost::json;
@@ -57,7 +61,9 @@ generateDefaultCacheConfig()
{"cache.num_cursors_from_diff", ConfigValue{ConfigType::Integer}.defaultValue(0)},
{"cache.num_cursors_from_account", ConfigValue{ConfigType::Integer}.defaultValue(0)},
{"cache.page_fetch_size", ConfigValue{ConfigType::Integer}.defaultValue(512)},
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async")}}
{"cache.load", ConfigValue{ConfigType::String}.defaultValue("async")},
{"cache.file.path", ConfigValue{ConfigType::String}.optional()},
{"cache.file.max_sequence_age", ConfigValue{ConfigType::Integer}.defaultValue(10)}}
};
}
@@ -90,18 +96,90 @@ INSTANTIATE_TEST_CASE_P(
CacheLoaderTest,
ParametrizedCacheLoaderTest,
Values(
Settings{.numCacheDiffs = 32, .numCacheMarkers = 48, .cachePageFetchSize = 512, .numThreads = 2},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 48, .cachePageFetchSize = 512, .numThreads = 4},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 48, .cachePageFetchSize = 512, .numThreads = 8},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 48, .cachePageFetchSize = 512, .numThreads = 16},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 128, .cachePageFetchSize = 24, .numThreads = 2},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 64, .cachePageFetchSize = 48, .numThreads = 4},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 48, .cachePageFetchSize = 64, .numThreads = 8},
Settings{.numCacheDiffs = 32, .numCacheMarkers = 24, .cachePageFetchSize = 128, .numThreads = 16},
Settings{.numCacheDiffs = 128, .numCacheMarkers = 128, .cachePageFetchSize = 24, .numThreads = 2},
Settings{.numCacheDiffs = 1024, .numCacheMarkers = 64, .cachePageFetchSize = 48, .numThreads = 4},
Settings{.numCacheDiffs = 512, .numCacheMarkers = 48, .cachePageFetchSize = 64, .numThreads = 8},
Settings{.numCacheDiffs = 64, .numCacheMarkers = 24, .cachePageFetchSize = 128, .numThreads = 16}
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 48,
.cachePageFetchSize = 512,
.numThreads = 2,
.cacheFileSettings = std::nullopt,
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 48,
.cachePageFetchSize = 512,
.numThreads = 4,
.cacheFileSettings = std::nullopt,
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 48,
.cachePageFetchSize = 512,
.numThreads = 8,
.cacheFileSettings = std::nullopt,
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 48,
.cachePageFetchSize = 512,
.numThreads = 16,
.cacheFileSettings = std::nullopt,
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 128,
.cachePageFetchSize = 24,
.numThreads = 2,
.cacheFileSettings = std::nullopt,
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 64,
.cachePageFetchSize = 48,
.numThreads = 4,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 48,
.cachePageFetchSize = 64,
.numThreads = 8,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 32,
.numCacheMarkers = 24,
.cachePageFetchSize = 128,
.numThreads = 16,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 128,
.numCacheMarkers = 128,
.cachePageFetchSize = 24,
.numThreads = 2,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 1024,
.numCacheMarkers = 64,
.cachePageFetchSize = 48,
.numThreads = 4,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 512,
.numCacheMarkers = 48,
.cachePageFetchSize = 64,
.numThreads = 8,
.cacheFileSettings = std::nullopt
},
Settings{
.numCacheDiffs = 64,
.numCacheMarkers = 24,
.cachePageFetchSize = 128,
.numThreads = 16,
.cacheFileSettings = std::nullopt
}
),
[](auto const& info) {
auto const settings = info.param;
@@ -285,3 +363,102 @@ TEST_F(CacheLoaderTest, DisabledCacheLoaderCanCallStopAndWait)
EXPECT_NO_THROW(loader.stop());
EXPECT_NO_THROW(loader.wait());
}
struct CacheLoaderFromFileTest : CacheLoaderTest {
CacheLoaderFromFileTest()
{
backend_->setRange(kSEQ - 20, kSEQ);
}
std::string const filePath = "./cache.bin";
uint32_t const maxSequenceLag = 10;
ClioConfigDefinition const cfg = getParseCacheConfig(
json::parse(
fmt::format(
R"JSON({{"cache": {{"load": "sync", "file": {{"path": "{}", "max_sequence_age": {}}}}}}})JSON",
filePath,
maxSequenceLag
)
)
);
CacheLoader<> loader{cfg, backend_, cache};
};
TEST_F(CacheLoaderFromFileTest, Success)
{
constexpr uint32_t kLOADED_SEQ = 12345;
EXPECT_CALL(cache, isFull).WillOnce(Return(false));
EXPECT_CALL(cache, loadFromFile(filePath, kSEQ - maxSequenceLag))
.WillOnce(Return(std::expected<void, std::string>{}));
EXPECT_CALL(cache, latestLedgerSequence).WillOnce(Return(kLOADED_SEQ));
loader.load(kSEQ);
std::optional<LedgerRange> const expectedLedgerRange =
LedgerRange{.minSequence = kSEQ - 20, .maxSequence = kLOADED_SEQ};
EXPECT_EQ(backend_->fetchLedgerRange(), expectedLedgerRange);
}
TEST_F(CacheLoaderFromFileTest, FailureBackToNormalLoad)
{
auto const diffs = diffProvider.getLatestDiff();
auto const loops = diffs.size() + 1;
auto const keysSize = 14;
EXPECT_CALL(cache, loadFromFile(filePath, kSEQ - maxSequenceLag))
.WillOnce(Return(std::expected<void, std::string>(std::unexpected("File not found"))));
EXPECT_CALL(*backend_, fetchLedgerDiff(_, _)).Times(32).WillRepeatedly(Return(diffs));
EXPECT_CALL(*backend_, doFetchSuccessorKey).Times(keysSize * loops).WillRepeatedly([this]() {
return diffProvider.nextKey(keysSize);
});
EXPECT_CALL(*backend_, doFetchLedgerObjects(_, kSEQ, _))
.Times(loops)
.WillRepeatedly(Return(std::vector<Blob>{keysSize - 1, Blob{'s'}}));
EXPECT_CALL(cache, isDisabled).WillRepeatedly(Return(false));
EXPECT_CALL(cache, updateImp).Times(loops);
EXPECT_CALL(cache, isFull).WillOnce(Return(false)).WillRepeatedly(Return(true));
EXPECT_CALL(cache, setFull).Times(1);
loader.load(kSEQ);
}
TEST_F(CacheLoaderFromFileTest, DontLoadWhenCacheIsDisabled)
{
auto const disabledCacheCfg =
getParseCacheConfig(json::parse(R"JSON({"cache": {"load": "none", "file": {"path": "/tmp/cache.bin"}}})JSON"));
CacheLoader loaderWithCacheDisabled{disabledCacheCfg, backend_, cache};
EXPECT_CALL(cache, isFull).WillOnce(Return(false));
EXPECT_CALL(cache, setDisabled);
loaderWithCacheDisabled.load(kSEQ);
}
TEST_F(CacheLoaderFromFileTest, MaxSequenceLagCalculation)
{
constexpr uint32_t kLOADED_SEQ = 12345;
EXPECT_CALL(cache, isFull).WillOnce(Return(false));
EXPECT_CALL(cache, loadFromFile(filePath, kSEQ - maxSequenceLag))
.WillOnce(Return(std::expected<void, std::string>{}));
EXPECT_CALL(cache, latestLedgerSequence).WillOnce(Return(kLOADED_SEQ));
loader.load(kSEQ);
}
TEST_F(CacheLoaderFromFileTest, MaxSequenceLagClampedToMinOfLedgerRange)
{
uint32_t const currentSeq = 110;
uint32_t const minSeq = currentSeq - maxSequenceLag + 10;
backend_->setRange(minSeq, currentSeq, true);
EXPECT_CALL(cache, isFull).WillOnce(Return(false));
EXPECT_CALL(cache, loadFromFile(filePath, minSeq)).WillOnce(Return(std::expected<void, std::string>{}));
EXPECT_CALL(cache, latestLedgerSequence).WillOnce(Return(minSeq + 1));
loader.load(currentSeq);
}

356
tests/unit/etlng/LedgerPublisherTests.cpp Normal file
View File

@@ -0,0 +1,356 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etl/SystemState.hpp"
#include "util/AsioContextTestFixture.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/MockSubscriptionManager.hpp"
#include "util/TestObject.hpp"
#include "util/config/ConfigDefinition.hpp"
#include <etlng/impl/LedgerPublisher.hpp>
#include <fmt/format.h>
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/chrono.h>
#include <xrpl/protocol/Indexes.h>
#include <xrpl/protocol/LedgerHeader.h>
#include <chrono>
#include <optional>
#include <vector>
using namespace testing;
using namespace etlng;
using namespace data;
using namespace std::chrono;
namespace {
constexpr auto kACCOUNT = "rf1BiGeXwwQoi8Z2ueFYTEXSwuJYfV2Jpn";
constexpr auto kACCOUNT2 = "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun";
constexpr auto kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constexpr auto kSEQ = 30;
constexpr auto kAGE = 800;
constexpr auto kAMOUNT = 100;
constexpr auto kFEE = 3;
constexpr auto kFINAL_BALANCE = 110;
constexpr auto kFINAL_BALANCE2 = 30;
MATCHER_P(ledgerHeaderMatcher, expectedHeader, "Headers match")
{
return arg.seq == expectedHeader.seq && arg.hash == expectedHeader.hash &&
arg.closeTime == expectedHeader.closeTime;
}
} // namespace
struct ETLLedgerPublisherNgTest : util::prometheus::WithPrometheus, MockBackendTestStrict, SyncAsioContextTest {
util::config::ClioConfigDefinition cfg{{}};
StrictMockSubscriptionManagerSharedPtr mockSubscriptionManagerPtr;
};
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderSkipDueToAge)
{
// Use kAGE (800) which is > MAX_LEDGER_AGE_SECONDS (600) to test skipping
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
auto dummyState = etl::SystemState{};
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
// Since age > MAX_LEDGER_AGE_SECONDS, these should not be called
EXPECT_CALL(*backend_, doFetchLedgerObject).Times(0);
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderWithinAgeLimit)
{
// Use age 0 which is < MAX_LEDGER_AGE_SECONDS to ensure publishing happens
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto dummyState = etl::SystemState{};
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
// Verify last published sequence is set immediately
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 0));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderIsWritingTrue)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
ctx_.run();
EXPECT_FALSE(backend_->fetchLedgerRange());
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderInRange)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger).WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 1));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
// mock 1 transaction
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerHeaderCloseTimeGreaterThanNow)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const nowPlus10 = system_clock::now() + seconds(10);
auto const closeTime = duration_cast<seconds>(nowPlus10.time_since_epoch()).count() - kRIPPLE_EPOCH_START;
dummyLedgerHeader.closeTime = ripple::NetClock::time_point{seconds{closeTime}};
backend_->setRange(kSEQ - 1, kSEQ);
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 1));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqStopIsTrue)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = true;
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
EXPECT_FALSE(publisher.publish(kSEQ, {}));
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqMaxAttempt)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
static constexpr auto kMAX_ATTEMPT = 2;
LedgerRange const range{.minSequence = kSEQ - 1, .maxSequence = kSEQ - 1};
EXPECT_CALL(*backend_, hardFetchLedgerRange).Times(kMAX_ATTEMPT).WillRepeatedly(Return(range));
EXPECT_FALSE(publisher.publish(kSEQ, kMAX_ATTEMPT, std::chrono::milliseconds{1}));
}
TEST_F(ETLLedgerPublisherNgTest, PublishLedgerSeqStopIsFalse)
{
auto dummyState = etl::SystemState{};
dummyState.isStopping = false;
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
LedgerRange const range{.minSequence = kSEQ, .maxSequence = kSEQ};
EXPECT_CALL(*backend_, hardFetchLedgerRange).WillOnce(Return(range));
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, kAGE);
EXPECT_CALL(*backend_, fetchLedgerBySequence(kSEQ, _)).WillOnce(Return(dummyLedgerHeader));
EXPECT_TRUE(publisher.publish(kSEQ, {}));
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishMultipleTxInOrder)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 0); // age is 0
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
// t1 index > t2 index
TransactionAndMetadata t1;
t1.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t1.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 2)
.getSerializer()
.peekData();
t1.ledgerSequence = kSEQ;
t1.date = 1;
TransactionAndMetadata t2;
t2.transaction =
createPaymentTransactionObject(kACCOUNT, kACCOUNT2, kAMOUNT, kFEE, kSEQ).getSerializer().peekData();
t2.metadata = createPaymentTransactionMetaObject(kACCOUNT, kACCOUNT2, kFINAL_BALANCE, kFINAL_BALANCE2, 1)
.getSerializer()
.peekData();
t2.ledgerSequence = kSEQ;
t2.date = 2;
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{t1, t2}));
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(_, _, fmt::format("{}-{}", kSEQ - 1, kSEQ), 2));
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges);
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t2, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction(t1, _)).InSequence(s);
ctx_.run();
EXPECT_TRUE(publisher.lastPublishAgeSeconds() <= 1);
}
TEST_F(ETLLedgerPublisherNgTest, PublishVeryOldLedgerShouldSkip)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
// Create a ledger header with age (800) greater than MAX_LEDGER_AGE_SECONDS (600)
auto const dummyLedgerHeader = createLedgerHeader(kLEDGER_HASH, kSEQ, 800);
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ);
publisher.publish(dummyLedgerHeader);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges).Times(0);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubTransaction).Times(0);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ);
ctx_.run();
}
TEST_F(ETLLedgerPublisherNgTest, PublishMultipleLedgersInQuickSuccession)
{
auto dummyState = etl::SystemState{};
dummyState.isWriting = true;
auto const dummyLedgerHeader1 = createLedgerHeader(kLEDGER_HASH, kSEQ, 0);
auto const dummyLedgerHeader2 = createLedgerHeader(kLEDGER_HASH, kSEQ + 1, 0);
auto publisher = etlng::impl::LedgerPublisher(ctx_, backend_, mockSubscriptionManagerPtr, dummyState);
backend_->setRange(kSEQ - 1, kSEQ + 1);
// Publish two ledgers in quick succession
publisher.publish(dummyLedgerHeader1);
publisher.publish(dummyLedgerHeader2);
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, doFetchLedgerObject(ripple::keylet::fees().key, kSEQ + 1, _))
.WillOnce(Return(createLegacyFeeSettingBlob(1, 2, 3, 4, 0)));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
EXPECT_CALL(*backend_, fetchAllTransactionsInLedger(kSEQ + 1, _))
.WillOnce(Return(std::vector<TransactionAndMetadata>{}));
Sequence const s;
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader1), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader1), _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubLedger(ledgerHeaderMatcher(dummyLedgerHeader2), _, _, _)).InSequence(s);
EXPECT_CALL(*mockSubscriptionManagerPtr, pubBookChanges(ledgerHeaderMatcher(dummyLedgerHeader2), _)).InSequence(s);
EXPECT_TRUE(publisher.getLastPublishedSequence());
EXPECT_EQ(publisher.getLastPublishedSequence().value(), kSEQ + 1);
ctx_.run();
}

61
tests/unit/util/ShasumTests.cpp
View File

@@ -22,6 +22,10 @@
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <string>
#include <utility>
using namespace util;
struct ShasumTest : testing::Test {
@@ -45,3 +49,60 @@ TEST_F(ShasumTest, sha256sumString)
EXPECT_EQ(sha256sumString(""), kEMPTY_HASH);
EXPECT_EQ(sha256sumString("hello world"), kHELLO_WORLD_HASH);
}
TEST_F(ShasumTest, Sha256sumStreamingEmpty)
{
Sha256sum hasher;
auto result = std::move(hasher).finalize();
ripple::uint256 expected;
ASSERT_TRUE(expected.parseHex(kEMPTY_HASH));
EXPECT_EQ(result, expected);
}
TEST_F(ShasumTest, Sha256sumStreamingSingleUpdate)
{
Sha256sum hasher;
std::string data = "hello world";
hasher.update(data.data(), data.size());
auto result = std::move(hasher).finalize();
ripple::uint256 expected;
ASSERT_TRUE(expected.parseHex(kHELLO_WORLD_HASH));
EXPECT_EQ(result, expected);
}
TEST_F(ShasumTest, Sha256sumStreamingMultipleUpdates)
{
Sha256sum hasher;
hasher.update("hello", 5);
hasher.update(" ", 1);
hasher.update("world", 5);
auto result = std::move(hasher).finalize();
ripple::uint256 expected;
ASSERT_TRUE(expected.parseHex(kHELLO_WORLD_HASH));
EXPECT_EQ(result, expected);
}
TEST_F(ShasumTest, Sha256sumUpdateTemplate)
{
Sha256sum hasher;
uint32_t value32 = 0x12345678;
uint64_t value64 = 0x123456789ABCDEF0;
hasher.update(value32);
hasher.update(value64);
auto result1 = std::move(hasher).finalize();
// Verify same result with raw data
Sha256sum hasher2;
hasher2.update(&value32, sizeof(value32));
hasher2.update(&value64, sizeof(value64));
auto result2 = std::move(hasher2).finalize();
EXPECT_EQ(result1, result2);
}