Compare commits

..

10 Commits

Author SHA1 Message Date
github-actions[bot]
e61ee30180 style: clang-tidy auto fixes (#1985)
Fixes #1984.

Co-authored-by: godexsoft <385326+godexsoft@users.noreply.github.com>
2025-04-02 12:40:38 +01:00
Sergey Kuznetsov
d3df6d10e4 fix: Fix ssl in new webserver (#1981)
Fixes #1980.

An SSL handshake was missing and WsConnection should be build from
stream not socket because in case SSL connection stream already
completed SSL handshake.
2025-04-01 16:12:16 +01:00
Sergey Kuznetsov
60df3a1914 ci: Pin cmake 3.31.6 for macos runners (#1983)
Fixes #1982.
2025-04-01 14:49:17 +01:00
cyan317
f454076fb6 feat: Snapshot import feature (#1970)
Implement snapshot import cmd
`clio_snapshot --server --grpc_server 0.0.0.0:12345 --path
<snapshot_path>`

Implement snapshot range cmd
`./clio_snapshot --range --path <snapshot_path>`

Add
LedgerHouses: It is responsible for reading/writing snapshot data
Server: Start grpc server and ws server
2025-03-26 09:11:15 +00:00
github-actions[bot]
66b3f40268 style: clang-tidy auto fixes (#1972)
Fixes #1971. 
Co-authored-by: godexsoft <385326+godexsoft@users.noreply.github.com>
2025-03-24 12:14:38 +00:00
Alex Kremer
b31b7633c9 feat: ETLng extensions (#1967)
For #1599 #1597
2025-03-21 16:41:29 +00:00
Peter Chen
a36aa3618f fix: ripple_flag logic in account lines (#1969)
fixes #1968
2025-03-19 10:26:04 -04:00
Sergey Kuznetsov
7943f47939 chore: Add git-cliff config (#1965)
First step for #1779.
2025-03-18 15:12:49 +00:00
Sergey Kuznetsov
67e451ec23 chore: Upgrade libxrpl to 2.4.0 (#1961) 2025-03-13 15:42:20 +00:00
github-actions[bot]
92789d5a91 style: clang-tidy auto fixes (#1963)
Fixes #1962.

Co-authored-by: godexsoft <385326+godexsoft@users.noreply.github.com>
2025-03-13 11:24:12 +00:00
62 changed files with 3243 additions and 496 deletions

View File

@@ -11,9 +11,35 @@ runs:
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates
brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates
echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
- name: Install CMake 3.31.6 on mac
if: ${{ runner.os == 'macOS' }}
shell: bash
run: |
# Uninstall any existing cmake
brew uninstall cmake --ignore-dependencies || true
# Download specific cmake formula
FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
mkdir -p /tmp/homebrew-formula
curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
# Verify the downloaded formula
ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1)
if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then
echo "Error: Formula checksum mismatch"
echo "Expected: $FORMULA_EXPECTED_SHA256"
echo "Actual: $ACTUAL_SHA256"
exit 1
fi
# Install cmake from the specific formula with force flag
brew install --force /tmp/homebrew-formula/cmake.rb
- name: Fix git permissions on Linux
if: ${{ runner.os == 'Linux' }}
shell: bash

92
cliff.toml Normal file
View File

@@ -0,0 +1,92 @@
# git-cliff ~ default configuration file
# https://git-cliff.org/docs/configuration
#
# Lines starting with "#" are comments.
# Configuration options are organized into tables and keys.
# See documentation for more information on available options.
[changelog]
# template for the changelog header
header = """
# Changelog\n
All notable changes to this project will be documented in this file.\n
"""
# template for the changelog body
# https://keats.github.io/tera/docs/#introduction
body = """
{% if version %}\
## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
{% else %}\
## [unreleased]
{% endif %}\
{% for group, commits in commits | filter(attribute="merge_commit", value=false) | group_by(attribute="group") %}
### {{ group | striptags | trim | upper_first }}
{% for commit in commits %}
- {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
{% if commit.breaking %}[**breaking**] {% endif %}\
{{ commit.message | upper_first }} {% if commit.remote.username %}by @{{ commit.remote.username }}{% endif %}\
{% endfor %}
{% endfor %}\n
"""
# template for the changelog footer
footer = """
<!-- generated by git-cliff -->
"""
# remove the leading and trailing s
trim = true
# postprocessors
postprocessors = [
# { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
]
# render body even when there are no releases to process
# render_always = true
# output file path
output = "CHANGELOG.md"
[git]
# parse the commits based on https://www.conventionalcommits.org
conventional_commits = true
# filter out the commits that are not conventional
filter_unconventional = true
# process each line of a commit as an individual commit
split_commits = false
# regex for preprocessing the commit messages
commit_preprocessors = [
# Replace issue numbers
#{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
# Check spelling of the commit with https://github.com/crate-ci/typos
# If the spelling is incorrect, it will be automatically fixed.
#{ pattern = '.*', replace_command = 'typos --write-changes -' },
]
# regex for parsing and grouping commits
commit_parsers = [
{ message = "^feat", group = "<!-- 0 -->🚀 Features" },
{ message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
{ message = "^doc", group = "<!-- 3 -->📚 Documentation" },
{ message = "^perf", group = "<!-- 4 -->⚡ Performance" },
{ message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
{ message = "^style.*[Cc]lang-tidy auto fixes", skip = true },
{ message = "^style", group = "<!-- 5 -->🎨 Styling" },
{ message = "^test", group = "<!-- 6 -->🧪 Testing" },
{ message = "^chore\\(release\\): prepare for", skip = true },
{ message = "^chore: Commits", skip = true },
{ message = "^chore\\(deps.*\\)", skip = true },
{ message = "^chore\\(pr\\)", skip = true },
{ message = "^chore\\(pull\\)", skip = true },
{ message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
{ body = ".*security", group = "<!-- 8 -->🛡️ Security" },
{ message = "^revert", group = "<!-- 9 -->◀️ Revert" },
{ message = ".*", group = "<!-- 10 -->💼 Other" },
]
# filter out the commits that are not matched by commit parsers
filter_commits = false
# sort the tags topologically
topo_order = false
# sort the commits inside sections by oldest/newest order
sort_commits = "oldest"
ignore_tags = "^.*-[b|rc].*"
[remote.github]
owner = "XRPLF"
repo = "clio"

View File

@@ -23,13 +23,13 @@ class Clio(ConanFile):
}
requires = [
'boost/1.82.0',
'boost/1.83.0',
'cassandra-cpp-driver/2.17.0',
'fmt/10.1.1',
'protobuf/3.21.9',
'grpc/1.50.1',
'openssl/1.1.1u',
'xrpl/2.4.0-rc4',
'openssl/1.1.1v',
'xrpl/2.4.0',
'zlib/1.3.1',
'libbacktrace/cci.20210118'
]

View File

@@ -154,7 +154,7 @@ public:
}
virtual ~BackendInterface() = default;
// TODO: Remove this hack once old ETL is removed.
// TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
// Cache should not be exposed thru BackendInterface
/**
@@ -648,6 +648,14 @@ public:
virtual void
writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
/**
* @brief Write a new account transaction.
*
* @param record An object representing the account transaction
*/
virtual void
writeAccountTransaction(AccountTransactionsData record) = 0;
/**
* @brief Write NFTs transactions.
*

View File

@@ -46,6 +46,7 @@
#include <xrpl/protocol/LedgerHeader.h>
#include <xrpl/protocol/nft.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstddef>
@@ -906,30 +907,42 @@ public:
statements.reserve(data.size() * 10); // assume 10 transactions avg
for (auto& record : data) {
std::transform(
std::begin(record.accounts),
std::end(record.accounts),
std::back_inserter(statements),
[this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
}
);
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
}
executor_.write(std::move(statements));
}
void
writeAccountTransaction(AccountTransactionsData record) override
{
std::vector<Statement> statements;
statements.reserve(record.accounts.size());
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
return schema_->insertAccountTx.bind(
std::forward<decltype(account)>(account),
std::make_tuple(record.ledgerSequence, record.transactionIndex),
record.txHash
);
});
executor_.write(std::move(statements));
}
void
writeNFTTransactions(std::vector<NFTTransactionsData> const& data) override
{
std::vector<Statement> statements;
statements.reserve(data.size());
std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) {
return schema_->insertNFTTx.bind(
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
);
@@ -999,7 +1012,7 @@ public:
std::vector<Statement> statements;
statements.reserve(data.size());
for (auto [mptId, holder] : data)
statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
statements.push_back(schema_->insertMPTHolder.bind(mptId, holder));
executor_.write(std::move(statements));
}

View File

@@ -54,7 +54,7 @@ struct AccountTransactionsData {
* @param meta The transaction metadata
* @param txHash The transaction hash
*/
AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash)
AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash)
: accounts(meta.getAffectedAccounts())
, ledgerSequence(meta.getLgrSeq())
, transactionIndex(meta.getIndex())

View File

@@ -20,6 +20,7 @@
#include "data/LedgerCache.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include <xrpl/basics/base_uint.h>
@@ -87,6 +88,42 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
}
}
void
LedgerCache::update(std::vector<etlng::model::Object> const& objs, uint32_t seq)
{
if (disabled_)
return;
std::scoped_lock const lck{mtx_};
if (seq > latestSeq_) {
ASSERT(
seq == latestSeq_ + 1 || latestSeq_ == 0,
"New sequence must be either next or first. seq = {}, latestSeq_ = {}",
seq,
latestSeq_
);
latestSeq_ = seq;
}
deleted_.clear(); // previous update's deletes no longer needed
for (auto const& obj : objs) {
if (!obj.data.empty()) {
auto& e = map_[obj.key];
if (seq > e.seq)
e = {.seq = seq, .blob = obj.data};
} else {
if (map_.contains(obj.key))
deleted_[obj.key] = map_[obj.key];
map_.erase(obj.key);
if (!full_)
deletes_.insert(obj.key);
}
}
cv_.notify_all();
}
std::optional<LedgerObject>
LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
{
@@ -139,6 +176,29 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
return {e->second.blob};
}
std::optional<Blob>
LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const
{
if (disabled_)
return std::nullopt;
std::shared_lock const lck{mtx_};
if (seq > latestSeq_)
return std::nullopt;
++objectReqCounter_.get();
auto e = deleted_.find(key);
if (e == deleted_.end())
return std::nullopt;
if (seq < e->second.seq)
return std::nullopt;
++objectHitCounter_.get();
return {e->second.blob};
}
void
LedgerCache::setDisabled()
{

View File

@@ -21,6 +21,7 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/prometheus/Bool.hpp"
#include "util/prometheus/Counter.hpp"
#include "util/prometheus/Label.hpp"
@@ -29,7 +30,6 @@
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/hardened_hash.h>
#include <atomic>
#include <condition_variable>
#include <cstddef>
#include <cstdint>
@@ -74,6 +74,7 @@ class LedgerCache : public LedgerCacheInterface {
)};
std::map<ripple::uint256, CacheEntry> map_;
std::map<ripple::uint256, CacheEntry> deleted_;
mutable std::shared_mutex mtx_;
std::condition_variable_any cv_;
@@ -94,11 +95,17 @@ class LedgerCache : public LedgerCacheInterface {
public:
void
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) override;
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground) override;
void
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) override;
std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const override;
std::optional<Blob>
getDeleted(ripple::uint256 const& key, uint32_t seq) const override;
std::optional<LedgerObject>
getSuccessor(ripple::uint256 const& key, uint32_t seq) const override;

View File

@@ -20,6 +20,7 @@
#pragma once
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/hardened_hash.h>
@@ -55,6 +56,15 @@ public:
virtual void
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) = 0;
/**
* @brief Update the cache with new ledger objects.
*
* @param objs The ledger objects to update cache with
* @param seq The sequence to update cache for
*/
virtual void
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) = 0;
/**
* @brief Fetch a cached object by its key and sequence number.
*
@@ -65,6 +75,16 @@ public:
virtual std::optional<Blob>
get(ripple::uint256 const& key, uint32_t seq) const = 0;
/**
* @brief Fetch a recently deleted object by its key and sequence number.
*
* @param key The key to fetch for
* @param seq The sequence to fetch for
* @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned
*/
virtual std::optional<Blob>
getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0;
/**
* @brief Gets a cached successor.
*

View File

@@ -20,7 +20,6 @@
#include "etl/ETLService.hpp"
#include "data/BackendInterface.hpp"
#include "data/LedgerCache.hpp"
#include "etl/CorruptionDetector.hpp"
#include "etl/NetworkValidatedLedgersInterface.hpp"
#include "feed/SubscriptionManagerInterface.hpp"

View File

@@ -31,15 +31,12 @@
namespace etl {
std::optional<ETLState>
tag_invoke(boost::json::value_to_tag<std::optional<ETLState>>, boost::json::value const& jv)
ETLState
tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv)
{
ETLState state;
auto const& jsonObject = jv.as_object();
if (jsonObject.contains(JS(error)))
return std::nullopt;
if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) {
auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object();
if (rippledInfo.contains(JS(network_id)))

View File

@@ -20,12 +20,14 @@
#pragma once
#include "data/BackendInterface.hpp"
#include "rpc/JS.hpp"
#include <boost/json.hpp>
#include <boost/json/conversion.hpp>
#include <boost/json/object.hpp>
#include <boost/json/value.hpp>
#include <boost/json/value_to.hpp>
#include <xrpl/protocol/jss.h>
#include <cstdint>
#include <optional>
@@ -54,8 +56,9 @@ struct ETLState {
return std::nullopt;
});
if (serverInfoRippled)
return boost::json::value_to<std::optional<ETLState>>(boost::json::value(*serverInfoRippled));
if (serverInfoRippled && not serverInfoRippled->contains(JS(error))) {
return boost::json::value_to<ETLState>(boost::json::value(*serverInfoRippled));
}
return std::nullopt;
}
@@ -67,7 +70,7 @@ struct ETLState {
* @param jv The json value to convert
* @return The ETLState
*/
std::optional<ETLState>
tag_invoke(boost::json::value_to_tag<std::optional<ETLState>>, boost::json::value const& jv);
ETLState
tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv);
} // namespace etl

View File

@@ -121,7 +121,7 @@ public:
LOG(log_.trace()) << "Inserting transaction = " << sttx.getTransactionID();
ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
ripple::TxMeta const txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx);
result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end());

View File

@@ -9,6 +9,10 @@ target_sources(
impl/Loading.cpp
impl/Monitor.cpp
impl/TaskManager.cpp
impl/ext/Cache.cpp
impl/ext/Core.cpp
impl/ext/NFT.cpp
impl/ext/Successor.cpp
)
target_link_libraries(clio_etlng PUBLIC clio_data)

View File

@@ -0,0 +1,59 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Cache.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <string>
#include <vector>
namespace etlng::impl {
CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache)
{
}
void
CacheExt::onLedgerData(model::LedgerData const& data) const
{
cache_.get().update(data.objects, data.seq);
LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size();
}
void
CacheExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size();
cache_.get().update(data.objects, data.seq);
cache_.get().setFull();
}
void
CacheExt::onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey)
const
{
LOG(log_.trace()) << "got initial objects cnt = " << objs.size();
cache_.get().update(objs, seq);
}
} // namespace etlng::impl

View File

@@ -0,0 +1,51 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/LedgerCacheInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <functional>
#include <string>
#include <vector>
namespace etlng::impl {
class CacheExt {
std::reference_wrapper<data::LedgerCacheInterface> cache_;
util::Logger log_{"ETL"};
public:
CacheExt(data::LedgerCacheInterface& cache);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,83 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Core.hpp"
#include "data/BackendInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
namespace etlng::impl {
CoreExt::CoreExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
{
}
void
CoreExt::onLedgerData(model::LedgerData const& data) const
{
LOG(log_.debug()) << "Loading ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
insertTransactions(data);
}
void
CoreExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.info()) << "Loading initial ledger data for " << data.seq;
backend_->writeLedger(data.header, auto{data.rawHeader});
insertTransactions(data);
}
void
CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::onObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq;
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
}
void
CoreExt::insertTransactions(model::LedgerData const& data) const
{
for (auto const& txn : data.transactions) {
LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID();
backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()});
backend_->writeTransaction(
auto{txn.key},
data.seq,
data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction'
auto{txn.raw},
auto{txn.metaRaw}
);
}
}
} // namespace etlng::impl

View File

@@ -0,0 +1,58 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <cstdint>
#include <memory>
namespace etlng::impl {
class CoreExt {
std::shared_ptr<BackendInterface> backend_;
util::Logger log_{"ETL"};
public:
CoreExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
void
onObject(uint32_t seq, model::Object const& obj) const;
private:
void
insertTransactions(model::LedgerData const& data) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,77 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/NFT.hpp"
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/NFTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace etlng::impl {
NFTExt::NFTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
{
}
void
NFTExt::onLedgerData(model::LedgerData const& data) const
{
writeNFTs(data);
}
void
NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const
{
LOG(log_.trace()) << "got initial object with key = " << obj.key;
backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw));
}
void
NFTExt::onInitialData(model::LedgerData const& data) const
{
LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
writeNFTs(data);
}
void
NFTExt::writeNFTs(model::LedgerData const& data) const
{
std::vector<NFTsData> nfts;
std::vector<NFTTransactionsData> nftTxs;
for (auto const& tx : data.transactions) {
auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx);
nftTxs.insert(nftTxs.end(), txs.begin(), txs.end());
if (maybeNFT)
nfts.push_back(*maybeNFT);
}
// This is uniqued so that we only write latest modification (as in previous implementation)
backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts));
backend_->writeNFTTransactions(nftTxs);
}
} // namespace etlng::impl

View File

@@ -0,0 +1,56 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "etl/NFTHelpers.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
namespace etlng::impl {
class NFTExt {
std::shared_ptr<BackendInterface> backend_;
util::Logger log_{"ETL"};
public:
NFTExt(std::shared_ptr<BackendInterface> backend);
void
onLedgerData(model::LedgerData const& data) const;
void
onInitialObject(uint32_t seq, model::Object const& obj) const;
void
onInitialData(model::LedgerData const& data) const;
private:
void
writeNFTs(model::LedgerData const& data) const;
};
} // namespace etlng::impl

View File

@@ -0,0 +1,222 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/impl/ext/Successor.hpp"
#include "data/BackendInterface.hpp"
#include "data/DBHelpers.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/Assert.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <cstdint>
#include <memory>
#include <optional>
#include <ranges>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace etlng::impl {
SuccessorExt::SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache)
: backend_(std::move(backend)), cache_(cache)
{
}
void
SuccessorExt::onInitialData(model::LedgerData const& data) const
{
ASSERT(cache_.get().isFull(), "Cache must be full at this point");
ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load");
ASSERT(data.objects.empty(), "Should not have objects from initial data");
writeSuccessors(data.seq);
writeEdgeKeys(data.seq, data.edgeKeys.value());
}
void
SuccessorExt::onInitialObjects(
uint32_t seq,
[[maybe_unused]] std::vector<model::Object> const& objs,
std::string lastKey
) const
{
for (auto const& obj : objs) {
if (!lastKey.empty())
backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw});
lastKey = obj.keyRaw;
}
}
void
SuccessorExt::onLedgerData(model::LedgerData const& data) const
{
namespace vs = std::views;
LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size()
<< "; got successors = " << data.successors.has_value() << "; cache is "
<< (cache_.get().isFull() ? "FULL" : "Not full");
auto filteredObjects = data.objects //
| vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; });
if (data.successors.has_value()) {
for (auto const& successor : data.successors.value())
writeIncludedSuccessor(data.seq, successor);
for (auto const& obj : filteredObjects)
writeIncludedSuccessor(data.seq, obj);
} else {
if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq)
throw std::logic_error("Cache is not full, but object neighbors were not included");
for (auto const& obj : filteredObjects)
updateSuccessorFromCache(data.seq, obj);
}
}
void
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const
{
auto firstBook = succ.firstBook;
if (firstBook.empty())
firstBook = uint256ToString(data::kLAST_KEY);
backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook));
}
void
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const
{
ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object");
// TODO: perhaps make these optionals inside of obj and move value_or here
auto pred = obj.predecessor;
auto succ = obj.successor;
if (obj.type == model::Object::ModType::Deleted) {
backend_->writeSuccessor(std::move(pred), seq, std::move(succ));
} else if (obj.type == model::Object::ModType::Created) {
backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw});
backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ));
}
}
void
SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const
{
auto const lb =
cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}});
auto const ub =
cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}});
auto checkBookBase = false;
auto const isDeleted = obj.data.empty();
if (isDeleted) {
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key));
} else {
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key));
backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key));
}
if (isDeleted) {
auto const old = cache_.get().getDeleted(obj.key, seq - 1);
ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key));
checkBookBase = isBookDir(obj.key, *old);
} else {
checkBookBase = isBookDir(obj.key, obj.data);
}
if (checkBookBase) {
auto const current = cache_.get().get(obj.key, seq);
auto const bookBase = getBookBase(obj.key);
if (isDeleted and not current.has_value()) {
updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase);
} else if (current.has_value()) {
auto const successor = cache_.get().getSuccessor(bookBase, seq);
ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq);
if (successor->key == obj.key) {
updateBookSuccessor(successor, seq, bookBase);
}
}
}
}
void
SuccessorExt::updateBookSuccessor(
std::optional<data::LedgerObject> const& maybeSuccessor,
auto seq,
ripple::uint256 const& bookBase
) const
{
if (maybeSuccessor.has_value()) {
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key));
} else {
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY));
}
}
void
SuccessorExt::writeSuccessors(uint32_t seq) const
{
ripple::uint256 prev = data::kFIRST_KEY;
while (auto cur = cache_.get().getSuccessor(prev, seq)) {
if (prev == data::kFIRST_KEY)
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key));
if (isBookDir(cur->key, cur->blob)) {
auto base = getBookBase(cur->key);
// make sure the base is not an actual object
if (not cache_.get().get(base, seq)) {
auto succ = cache_.get().getSuccessor(base, seq);
ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
if (succ->key == cur->key)
backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key));
}
}
prev = cur->key;
}
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY));
}
void
SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const
{
for (auto const& key : edgeKeys) {
auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq);
if (succ)
backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key));
}
}
} // namespace etlng::impl

View File

@@ -0,0 +1,82 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#pragma once
#include "data/BackendInterface.hpp"
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "util/log/Logger.hpp"
#include <xrpl/basics/base_uint.h>
#include <xrpl/basics/strHex.h>
#include <cstdint>
#include <functional>
#include <memory>
#include <optional>
#include <string>
#include <vector>
namespace etlng::impl {
class SuccessorExt {
std::shared_ptr<BackendInterface> backend_;
std::reference_wrapper<data::LedgerCacheInterface> cache_;
util::Logger log_{"ETL"};
public:
SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache);
void
onInitialData(model::LedgerData const& data) const;
void
onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector<model::Object> const& objs, std::string lastKey) const;
void
onLedgerData(model::LedgerData const& data) const;
private:
void
writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const;
void
writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const;
void
updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const;
void
updateBookSuccessor(
std::optional<data::LedgerObject> const& maybeSuccessor,
auto seq,
ripple::uint256 const& bookBase
) const;
void
writeSuccessors(uint32_t seq) const;
void
writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const;
};
} // namespace etlng::impl

View File

@@ -385,7 +385,7 @@ insertDeliverMaxAlias(boost::json::object& txJson, std::uint32_t const apiVersio
{
if (txJson.contains(JS(TransactionType)) and txJson.at(JS(TransactionType)).is_string() and
txJson.at(JS(TransactionType)).as_string() == JS(Payment) and txJson.contains(JS(Amount))) {
txJson[JS(DeliverMax)] = txJson[JS(Amount)];
txJson.insert_or_assign(JS(DeliverMax), txJson[JS(Amount)]);
if (apiVersion > 1)
txJson.erase(JS(Amount));
}

View File

@@ -102,6 +102,12 @@ AccountLinesHandler::addLine(
line.qualityIn = lineQualityIn;
line.qualityOut = lineQualityOut;
if (lineNoRipple)
line.noRipple = true;
if (lineNoRipplePeer)
line.noRipplePeer = true;
if (lineAuth)
line.authorized = true;
@@ -120,8 +126,6 @@ AccountLinesHandler::addLine(
if (lineDeepFreezePeer)
line.deepFreezePeer = true;
line.noRipple = lineNoRipple;
line.noRipplePeer = lineNoRipplePeer;
lines.push_back(line);
}
@@ -257,8 +261,11 @@ tag_invoke(
{JS(quality_out), line.qualityOut},
};
obj[JS(no_ripple)] = line.noRipple;
obj[JS(no_ripple_peer)] = line.noRipplePeer;
if (line.noRipple)
obj[JS(no_ripple)] = *(line.noRipple);
if (line.noRipplePeer)
obj[JS(no_ripple_peer)] = *(line.noRipplePeer);
if (line.authorized)
obj[JS(authorized)] = *(line.authorized);

View File

@@ -70,8 +70,8 @@ public:
std::string limitPeer;
uint32_t qualityIn{};
uint32_t qualityOut{};
bool noRipple{};
bool noRipplePeer{};
std::optional<bool> noRipple;
std::optional<bool> noRipplePeer;
std::optional<bool> authorized;
std::optional<bool> peerAuthorized;
std::optional<bool> freeze;

View File

@@ -10,7 +10,6 @@ target_sources(
ng/impl/ErrorHandling.cpp
ng/impl/ConnectionHandler.cpp
ng/impl/ServerSslContext.cpp
ng/impl/WsConnection.cpp
ng/Request.cpp
ng/Response.cpp
ng/Server.cpp

View File

@@ -46,6 +46,7 @@
#include <boost/system/system_error.hpp>
#include <fmt/core.h>
#include <chrono>
#include <cstddef>
#include <functional>
#include <memory>
@@ -136,13 +137,19 @@ makeConnection(
if (not sslContext.has_value())
return std::unexpected{"Error creating a connection: SSL is not supported by this server"};
connection = std::make_unique<impl::SslHttpConnection>(
auto sslConnection = std::make_unique<impl::SslHttpConnection>(
std::move(sslDetectionResult.socket),
std::move(ip),
std::move(sslDetectionResult.buffer),
*sslContext,
tagDecoratorFactory
);
sslConnection->setTimeout(std::chrono::seconds{10});
auto const maybeError = sslConnection->sslHandshake(yield);
if (maybeError.has_value())
return std::unexpected{fmt::format("SSL handshake error: {}", maybeError->message())};
connection = std::move(sslConnection);
} else {
connection = std::make_unique<impl::PlainHttpConnection>(
std::move(sslDetectionResult.socket),
@@ -164,7 +171,6 @@ makeConnection(
std::expected<ConnectionPtr, std::string>
tryUpgradeConnection(
impl::UpgradableConnectionPtr connection,
std::optional<boost::asio::ssl::context>& sslContext,
util::TagDecoratorFactory& tagDecoratorFactory,
boost::asio::yield_context yield
)
@@ -177,7 +183,7 @@ tryUpgradeConnection(
}
if (*expectedIsUpgrade) {
auto expectedUpgradedConnection = connection->upgrade(sslContext, tagDecoratorFactory, yield);
auto expectedUpgradedConnection = connection->upgrade(tagDecoratorFactory, yield);
if (expectedUpgradedConnection.has_value())
return std::move(expectedUpgradedConnection).value();
@@ -316,8 +322,7 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield
return;
}
auto connection =
tryUpgradeConnection(std::move(connectionExpected).value(), sslContext_, tagDecoratorFactory_, yield);
auto connection = tryUpgradeConnection(std::move(connectionExpected).value(), tagDecoratorFactory_, yield);
if (not connection.has_value()) {
LOG(log_.info()) << connection.error();
return;

View File

@@ -28,10 +28,12 @@
#include "web/ng/impl/Concepts.hpp"
#include "web/ng/impl/WsConnection.hpp"
#include <boost/asio/buffer.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/context.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/asio/ssl/stream_base.hpp>
#include <boost/beast/core/basic_stream.hpp>
#include <boost/beast/core/error.hpp>
#include <boost/beast/core/flat_buffer.hpp>
@@ -57,11 +59,7 @@ public:
isUpgradeRequested(boost::asio::yield_context yield) = 0;
virtual std::expected<ConnectionPtr, Error>
upgrade(
std::optional<boost::asio::ssl::context>& sslContext,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
) = 0;
upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) = 0;
virtual std::optional<Error>
sendRaw(
@@ -104,6 +102,22 @@ public:
{
}
std::optional<Error>
sslHandshake(boost::asio::yield_context yield)
requires IsSslTcpStream<StreamType>
{
boost::system::error_code error;
boost::beast::get_lowest_layer(stream_).expires_after(timeout_);
auto const bytesUsed =
stream_.async_handshake(boost::asio::ssl::stream_base::server, buffer_.cdata(), yield[error]);
if (error)
return error;
buffer_.consume(bytesUsed);
return std::nullopt;
}
bool
wasUpgraded() const override
{
@@ -183,35 +197,18 @@ public:
}
std::expected<ConnectionPtr, Error>
upgrade(
[[maybe_unused]] std::optional<boost::asio::ssl::context>& sslContext,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
) override
upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) override
{
ASSERT(request_.has_value(), "Request must be present to upgrade the connection");
if constexpr (IsSslTcpStream<StreamType>) {
ASSERT(sslContext.has_value(), "SSL context must be present to upgrade the connection");
return makeSslWsConnection(
boost::beast::get_lowest_layer(stream_).release_socket(),
std::move(ip_),
std::move(buffer_),
std::move(request_).value(),
sslContext.value(),
tagDecoratorFactory,
yield
);
} else {
return makePlainWsConnection(
stream_.release_socket(),
std::move(ip_),
std::move(buffer_),
std::move(request_).value(),
tagDecoratorFactory,
yield
);
}
return makeWsConnection(
std::move(stream_),
std::move(ip_),
std::move(buffer_),
std::move(request_).value(),
tagDecoratorFactory,
yield
);
}
private:

View File

@@ -1,77 +0,0 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2024, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "web/ng/impl/WsConnection.hpp"
#include "util/Taggable.hpp"
#include "web/ng/Error.hpp"
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/context.hpp>
#include <boost/beast/core/flat_buffer.hpp>
#include <boost/beast/http/message.hpp>
#include <boost/beast/http/string_body.hpp>
#include <memory>
#include <string>
#include <utility>
namespace web::ng::impl {
std::expected<std::unique_ptr<PlainWsConnection>, Error>
makePlainWsConnection(
boost::asio::ip::tcp::socket socket,
std::string ip,
boost::beast::flat_buffer buffer,
boost::beast::http::request<boost::beast::http::string_body> request,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
)
{
auto connection = std::make_unique<PlainWsConnection>(
std::move(socket), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
);
auto maybeError = connection->performHandshake(yield);
if (maybeError.has_value())
return std::unexpected{maybeError.value()};
return connection;
}
std::expected<std::unique_ptr<SslWsConnection>, Error>
makeSslWsConnection(
boost::asio::ip::tcp::socket socket,
std::string ip,
boost::beast::flat_buffer buffer,
boost::beast::http::request<boost::beast::http::string_body> request,
boost::asio::ssl::context& sslContext,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
)
{
auto connection = std::make_unique<SslWsConnection>(
std::move(socket), std::move(ip), std::move(buffer), sslContext, std::move(request), tagDecoratorFactory
);
auto maybeError = connection->performHandshake(yield);
if (maybeError.has_value())
return std::unexpected{maybeError.value()};
return connection;
}
} // namespace web::ng::impl

View File

@@ -68,31 +68,14 @@ class WsConnection : public WsConnectionBase {
public:
WsConnection(
boost::asio::ip::tcp::socket socket,
StreamType&& stream,
std::string ip,
boost::beast::flat_buffer buffer,
boost::beast::http::request<boost::beast::http::string_body> initialRequest,
util::TagDecoratorFactory const& tagDecoratorFactory
)
requires IsTcpStream<StreamType>
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
, stream_(std::move(socket))
, initialRequest_(std::move(initialRequest))
{
setupWsStream();
}
WsConnection(
boost::asio::ip::tcp::socket socket,
std::string ip,
boost::beast::flat_buffer buffer,
boost::asio::ssl::context& sslContext,
boost::beast::http::request<boost::beast::http::string_body> initialRequest,
util::TagDecoratorFactory const& tagDecoratorFactory
)
requires IsSslTcpStream<StreamType>
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
, stream_(std::move(socket), sslContext)
, stream_(std::move(stream))
, initialRequest_(std::move(initialRequest))
{
setupWsStream();
@@ -189,25 +172,24 @@ private:
using PlainWsConnection = WsConnection<boost::beast::tcp_stream>;
using SslWsConnection = WsConnection<boost::asio::ssl::stream<boost::beast::tcp_stream>>;
std::expected<std::unique_ptr<PlainWsConnection>, Error>
makePlainWsConnection(
boost::asio::ip::tcp::socket socket,
template <typename StreamType>
std::expected<std::unique_ptr<WsConnection<StreamType>>, Error>
makeWsConnection(
StreamType&& stream,
std::string ip,
boost::beast::flat_buffer buffer,
boost::beast::http::request<boost::beast::http::string_body> request,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
);
std::expected<std::unique_ptr<SslWsConnection>, Error>
makeSslWsConnection(
boost::asio::ip::tcp::socket socket,
std::string ip,
boost::beast::flat_buffer buffer,
boost::beast::http::request<boost::beast::http::string_body> request,
boost::asio::ssl::context& sslContext,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield
);
)
{
auto connection = std::make_unique<WsConnection<StreamType>>(
std::forward<StreamType>(stream), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
);
auto maybeError = connection->performHandshake(yield);
if (maybeError.has_value())
return std::unexpected{maybeError.value()};
return connection;
}
} // namespace web::ng::impl

View File

@@ -19,134 +19,31 @@
#include "util/BinaryTestObject.hpp"
#include "data/DBHelpers.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/Extraction.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gtest/gtest.h>
#include <org/xrpl/rpc/v1/ledger.pb.h>
#include <xrpl/basics/Blob.h>
#include <xrpl/basics/StringUtilities.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
#include <xrpl/protocol/AccountID.h>
#include <xrpl/protocol/STTx.h>
#include <xrpl/protocol/Serializer.h>
#include <xrpl/protocol/TxFormats.h>
#include <xrpl/protocol/TxMeta.h>
#include <optional>
#include <string>
#include <utility>
namespace {
constinit auto const kSEQ = 30;
constinit auto const kTXN_HEX =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
constinit auto const kTXN_META =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
constinit auto const kRAW_HEADER =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
@@ -159,27 +56,27 @@ constinit auto const kRAW_HEADER =
namespace util {
std::pair<std::string, std::string>
createNftTxAndMetaBlobs()
createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr)
{
return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)};
return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)};
}
std::pair<ripple::STTx, ripple::TxMeta>
createNftTxAndMeta()
createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr)
{
ripple::uint256 hash;
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
EXPECT_TRUE(hash.parseHex(hashStr));
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs();
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr);
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}};
}
etlng::model::Transaction
createTransaction(ripple::TxType type)
createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr)
{
auto const [sttx, meta] = createNftTxAndMeta();
auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr);
return {
.raw = "",
.metaRaw = "",
@@ -192,10 +89,9 @@ createTransaction(ripple::TxType type)
}
etlng::model::Object
createObject()
createObject(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
@@ -205,12 +101,62 @@ createObject()
"8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
return {
.key = {},
.keyRaw = hexStringToBinaryString(kOBJ_KEY),
.data = {},
.dataRaw = hexStringToBinaryString(kOBJ_BLOB),
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key)
{
// random object taken from initial ledger load
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
static constinit auto const kOBJ_BLOB =
"11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE"
"3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000"
"0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3";
return {
.key = binaryStringToUint256(hexStringToBinaryString(key)),
.keyRaw = hexStringToBinaryString(key),
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
.successor = hexStringToBinaryString(kOBJ_SUCC),
.predecessor = hexStringToBinaryString(kOBJ_PRED),
.type = modType,
};
}
etlng::model::Object
createObjectWithTwoNFTs()
{
std::string const url1 = "abcd1";
std::string const url2 = "abcd2";
ripple::Blob const uri1Blob(url1.begin(), url1.end());
ripple::Blob const uri2Blob(url2.begin(), url2.end());
constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q";
constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000";
constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA";
auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt);
auto const serializerNftPage = nftPage.getSerializer();
auto const account = getAccountIdWithString(kACCOUNT);
return {
.key = {},
.keyRaw = std::string(reinterpret_cast<char const*>(account.data()), ripple::AccountID::size()),
.data = {},
.dataRaw =
std::string(static_cast<char const*>(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()),
.successor = "",
.predecessor = "",
.type = etlng::model::Object::ModType::Created,
};
}
@@ -219,8 +165,10 @@ etlng::model::BookSuccessor
createSuccessor()
{
return {
.firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
.bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
.firstBook =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}),
.bookBase =
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}),
};
}

View File

@@ -32,17 +32,149 @@
namespace util {
static constexpr auto kDEFAULT_TXN_HEX =
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
"DE3538F248662FC73C";
static constexpr auto kDEFAULT_TXN_META =
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
"C200000028751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
"00000029751868747470733A2F2F677265677765697362726F642E636F"
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
"B100000015751868747470733A2F2F677265677765697362726F642E63"
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
"2FC73CE1E1F1031000";
static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E";
static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
[[maybe_unused, nodiscard]] std::pair<std::string, std::string>
createNftTxAndMetaBlobs();
createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX);
[[maybe_unused, nodiscard]] std::pair<ripple::STTx, ripple::TxMeta>
createNftTxAndMeta();
createNftTxAndMeta(
std::string hashStr = kDEFAULT_HASH,
std::string metaStr = kDEFAULT_TXN_META,
std::string txnStr = kDEFAULT_TXN_HEX
);
[[maybe_unused, nodiscard]] etlng::model::Transaction
createTransaction(ripple::TxType type);
createTransaction(
ripple::TxType type,
std::string hashStr = kDEFAULT_HASH,
std::string metaStr = kDEFAULT_TXN_META,
std::string txnStr = kDEFAULT_TXN_HEX
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObject();
createObject(
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
std::string key = kDEFAULT_OBJ_KEY
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObjectWithBookBase(
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
std::string key = kDEFAULT_OBJ_KEY
);
[[maybe_unused, nodiscard]] etlng::model::Object
createObjectWithTwoNFTs();
[[maybe_unused, nodiscard]] etlng::model::BookSuccessor
createSuccessor();

View File

@@ -203,6 +203,8 @@ struct MockBackend : public BackendInterface {
MOCK_METHOD(void, writeAccountTransactions, (std::vector<AccountTransactionsData>), (override));
MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override));
MOCK_METHOD(void, writeNFTTransactions, (std::vector<NFTTransactionsData> const&), (override));
MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override));

View File

@@ -21,6 +21,7 @@
#include "data/LedgerCacheInterface.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include <gmock/gmock.h>
#include <xrpl/basics/base_uint.h>
@@ -41,6 +42,10 @@ struct MockLedgerCache : data::LedgerCacheInterface {
MOCK_METHOD(std::optional<data::Blob>, get, (ripple::uint256 const& a, uint32_t b), (const, override));
MOCK_METHOD(void, update, (std::vector<etlng::model::Object> const&, uint32_t), (override));
MOCK_METHOD(std::optional<data::Blob>, getDeleted, (ripple::uint256 const&, uint32_t), (const, override));
MOCK_METHOD(
std::optional<data::LedgerObject>,
getSuccessor,

View File

@@ -67,9 +67,7 @@ struct MockHttpConnectionImpl : web::ng::impl::UpgradableConnection {
MOCK_METHOD(
UpgradeReturnType,
upgrade,
(OptionalSslContext & sslContext,
util::TagDecoratorFactory const& tagDecoratorFactory,
boost::asio::yield_context yield),
(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield),
(override)
);
};

View File

@@ -402,7 +402,7 @@ TEST_F(BackendCassandraTest, Basic)
ripple::uint256 hash256;
EXPECT_TRUE(hash256.parseHex(hashHex));
ripple::TxMeta txMeta{hash256, lgrInfoNext.seq, metaBlob};
ripple::TxMeta const txMeta{hash256, lgrInfoNext.seq, metaBlob};
auto accountsSet = txMeta.getAffectedAccounts();
for (auto& a : accountsSet) {
affectedAccounts.push_back(a);

View File

@@ -45,6 +45,10 @@ target_sources(
etlng/LoadingTests.cpp
etlng/NetworkValidatedLedgersTests.cpp
etlng/MonitorTests.cpp
etlng/ext/CoreTests.cpp
etlng/ext/CacheTests.cpp
etlng/ext/NFTTests.cpp
etlng/ext/SuccessorTests.cpp
# Feed
util/BytesConverterTests.cpp
feed/BookChangesFeedTests.cpp

View File

@@ -0,0 +1,91 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Cache.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const kUNUSED_LAST_KEY = "unused";
auto
createTestData()
{
auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = {},
.objects = std::move(objects),
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct CacheExtTests : util::prometheus::WithPrometheus {
protected:
MockLedgerCache cache_;
etlng::impl::CacheExt ext_{cache_};
};
TEST_F(CacheExtTests, OnLedgerDataUpdatesCache)
{
auto const data = createTestData();
EXPECT_CALL(cache_, update(data.objects, data.seq));
ext_.onLedgerData(data);
}
TEST_F(CacheExtTests, OnInitialDataUpdatesCacheAndSetsFull)
{
auto const data = createTestData();
EXPECT_CALL(cache_, update(data.objects, data.seq));
EXPECT_CALL(cache_, setFull);
ext_.onInitialData(data);
}
TEST_F(CacheExtTests, OnInitialObjectsUpdateCache)
{
auto const objects = std::vector{util::createObject(), util::createObject()};
EXPECT_CALL(cache_, update(objects, kSEQ));
ext_.onInitialObjects(kSEQ, objects, kUNUSED_LAST_KEY);
}

View File

@@ -0,0 +1,107 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Core.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/protocol/TxFormats.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto
createTestData()
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::CoreExt ext_{backend_};
};
TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
ext_.onLedgerData(data);
}
TEST_F(CoreExtTests, OnInitialDataWritesLedgerAndTransactions)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
ext_.onInitialData(data);
}
TEST_F(CoreExtTests, OnInitialObjectWritesLedgerObject)
{
auto const data = util::createObject();
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
ext_.onInitialObject(kSEQ, data);
}
TEST_F(CoreExtTests, OnObjectWritesLedgerObject)
{
auto const data = util::createObject();
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
ext_.onObject(kSEQ, data);
}

View File

@@ -0,0 +1,287 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "etlng/Models.hpp"
#include "etlng/impl/ext/NFT.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockPrometheus.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/protocol/TxFormats.h>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
constinit auto const kTXN_HEX2 =
"12001D230606B58324048A8B6F501C50E8EBCD412E6CF9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487E501DA2A1BC9A62AAEB2A"
"2A70F895587A3FB752514AA03F8C6E7C84864653B8673E0368400000000000001E60134000000000001F09732103B8C234E0598BC26D8A3E1B"
"FF53EB252EC0F15EA6800E4D85AA5D7CD15D76B01E744730450221009C0AFF5F3298E10ABE42894717DA46B59529A366527AA5DFC1577ADEA9"
"B20FA70220494D1D9BFEF2AB09F4D6403AAC5B9DCC5B859DCEC1380C6D817A6EDFE7E68FD581141565EED165BA79999425204A8491C73B1301"
"E34FF9EA7D0F7872702E63616665202D2073616C65E1F1";
constinit auto const kTXN_META2 =
"201C00000040F8E51100502505A59E11552ABC2FD74D879BE58489A588838AA2BA59E1E05A48A574226CD8B6CE77998971560B639A808E3B97"
"42A25334E5CF68EEDEDE52F54E50F4E63921C9F3C40588E426E6FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C"
"052AB2897542697066733A2F2F6261666B726569683673796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E71673472"
"6E747037653769757469766C796275E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D526950"
"5679335464654A6170697974576B686141576B4D716D39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60F"
"DDC47DABE4D52E4AD1F0A50A85CBBC00000022752E516D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169"
"316B4C316E5255797131E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C457664644634"
"5442427374455138484C66694867456B6A4465514D56676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4"
"D52E4AD1F0A5608E298B000000EF752E516D4E6E56566F4250426755783848334641486F436162593647347577445758767738757455636976"
"73764D7142E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646"
"777A526D517472694841663277437A3567786161355256446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0"
"A5B09C35A40000020A752E516D5168584346325738535571657633333657703572357935326141464D59564A667570534A536B6B425047786E"
"E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B6968756000001BC752E516D5359647963353276657A7478365338673231"
"70544651724E6E624762676A5A4B5047753836464231394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C"
"000000D2752E516D506645624A38446D624E69794C55637441796A3473625453515937314177597A32776A595238703838416432E1EC5A0008"
"27107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37"
"773338734147397469725867326D626D4262596771E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F75"
"2E516D505A733255393159764538374277635677587A63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E6"
"4C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E21584000001EA752E516D656366646769555A6B37355574667152763474696644557257345439"
"4571616F733231757036706E75345663E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573"
"614345725175396F48596A71765172564B31473155504657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE"
"11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A6854653776"
"43624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A2F2F6261"
"667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E6965"
"2F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F62616679"
"62656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E69652F36"
"342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D616D397436"
"5A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0FE30354A4"
"3281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D79756E697736"
"647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A43281629A"
"0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D6575326934"
"65726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A00082710"
"AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E61776833"
"6F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B7320"
"2331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F516D63436D"
"526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E72200000000501A0B639A808E"
"3B9742A25334E5CF68EEDEDE52F54E4A69498F5B79952C052AB289501B0B639A808E3B9742A25334E5CF68EEDEDE52F54E50F4E639664EC7E5"
"0588E658FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C052AB2897542697066733A2F2F6261666B7265696836"
"73796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E716734726E747037653769757469766C796275E1EC5A00082710"
"7B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D5269505679335464654A6170697974576B686141576B4D716D"
"39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A50A85CBBC00000022752E51"
"6D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169316B4C316E5255797131E1EC5A000827107B87E64C88"
"4BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C4576646446345442427374455138484C66694867456B6A4465514D56"
"676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5608E298B000000EF752E516D4E6E5656"
"6F4250426755783848334641486F43616259364734757744575876773875745563697673764D7142E1EC5A000827107B87E64C884BBFB60FDD"
"C47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646777A526D517472694841663277437A35677861613552"
"56446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B09C35A40000020A752E516D516858434632573853"
"5571657633333657703572357935326141464D59564A667570534A536B6B425047786EE1EC5A000827107B87E64C884BBFB60FDDC47DABE4D5"
"2E4AD1F0A5B6968756000001BC752E516D5359647963353276657A747836533867323170544651724E6E624762676A5A4B5047753836464231"
"394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C000000D2752E516D506645624A38446D624E69794C55"
"637441796A3473625453515937314177597A32776A595238703838416432E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5"
"CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37773338734147397469725867326D626D4262596771E1"
"EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F752E516D505A733255393159764538374277635677587A"
"63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E2158400"
"0001EA752E516D656366646769555A6B373555746671527634746966445572573454394571616F733231757036706E75345663E1EC5A000827"
"107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573614345725175396F48596A71765172564B3147315550"
"4657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051752E"
"516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44668A"
"2B96DFDE11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A68"
"5465377643624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A"
"2F2F6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A3670717136356635"
"6F6E69652F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F"
"6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E"
"69652F36342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D61"
"6D3974365A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0F"
"E30354A43281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D7975"
"6E697736647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A4"
"3281629A0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D65"
"7532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A"
"00082710AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E"
"617768336F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B73205075"
"6E6B73202331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F51"
"6D63436D526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E1E51100502505A4C9"
"C9557DF04CC68DE7C5EF20DBF705221EEDB05FE3806BC3F6A35240652C3E9C97BA5A56246B3E06AB367AB9614566B6F90C718B52A4440852A4"
"440804D409E004C90E52E6FAEC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478B72AD46100000008755E68747470733A2F2F69"
"7066732E696F2F697066732F62616679626569656F6C7667696F71766F737436346367646873797876726962336D6265797278773477626F61"
"617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10"
"276700000009755E68747470733A2F2F697066732E696F2F697066732F62616679626569686D6374376A766B7236366E67337975676D33706D"
"70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA70"
"69E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E68747470733A2F2F697066732E696F2F697066732F6261667962656968626D6D"
"6E626C687736656D776E6A733766787778376736376B6E6A626B6966636674787A6B66777632767162357A6C727575612F6D65746164617461"
"2E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478FBE441630000000B755E68747470733A2F2F697066732E696F"
"2F697066732F62616679626569686E6C77656C7965357270706963646761617678756869376D61636E697879627077667133796734626C3366"
"6A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051"
"752E516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44"
"668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B535470"
"31514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D51"
"674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE58"
"EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A7661"
"6D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC753569"
"7066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A0008"
"2710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F62"
"6D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC24F"
"1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C783579693375"
"706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408"
"005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D7547724839"
"4C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D555744524C"
"425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB9614566"
"B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E346958597233487341715771"
"673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD7535697066733A"
"2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A00081388246B"
"3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A51567957"
"4B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204C9"
"12F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E47"
"E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E357163334E"
"753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52"
"A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E654474"
"6656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D5779"
"6D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367AB9"
"614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A743147454564"
"4C787161513466714E637375363854677153E1F1E1E72200000000501A246B3E06AB367AB9614566B6F90C718B52A444084B237478B72AD461"
"00000008501B246B3E06AB367AB9614566B6F90C718B52A4440852A444080D2641FC04C91315FAEC5A00081388DAA8A3AA7069E65EC1E3E557"
"1D9B6F274B237478B72AD46100000008755E68747470733A2F2F697066732E696F2F697066732F62616679626569656F6C7667696F71766F73"
"7436346367646873797876726962336D6265797278773477626F61617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC"
"5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10276700000009755E68747470733A2F2F697066732E696F2F697066732F62"
"616679626569686D6374376A766B7236366E67337975676D33706D70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69"
"792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E6874747073"
"3A2F2F697066732E696F2F697066732F6261667962656968626D6D6E626C687736656D776E6A733766787778376736376B6E6A626B69666366"
"74787A6B66777632767162357A6C727575612F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B23"
"7478FBE441630000000B755E68747470733A2F2F697066732E696F2F697066732F62616679626569686E6C77656C7965357270706963646761"
"617678756869376D61636E697879627077667133796734626C33666A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A"
"44668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B5354"
"7031514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D"
"51674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE"
"58EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A76"
"616D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC7535"
"697066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A00"
"082710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F"
"626D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC2"
"4F1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C7835796933"
"75706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A444"
"08005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D75477248"
"394C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D55574452"
"4C425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB96145"
"66B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E3469585972334873417157"
"71673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD753569706673"
"3A2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A0008138824"
"6B3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A515679"
"574B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204"
"C912F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E"
"47E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E35716333"
"4E753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B"
"52A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E6544"
"746656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D57"
"796D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367A"
"B9614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A7431474545"
"644C787161513466714E637375363854677153E1F1E1E1E4110064562AED34CB796DF0E82AAC7EB958158EEBF99F51AA8C96B85654DDE05206"
"C18BBCE7220000000225059FB7B755172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235BEBBE582AED34CB796DF0E82A"
"AC7EB958158EEBF99F51AA8C96B85654DDE05206C18BBC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051E1"
"E1E51100612505A58BFB555F1D177245DF64F1BAC04B3EFF72C9B448710F95E86355339305264B0C6450345643CECFECC44660B31BEB70A4AE"
"78ED0BB4B31A754CE746848150EFFC03418847E62D0000010E624000000004FF530BE1E722000000002404C9421F2D0000010D202B00000712"
"202C00000493203204C90C7062400000000506A60B8114246B3E06AB367AB9614566B6F90C718B52A44408E1E1E41100375650E8EBCD412E6C"
"F9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487EE722000000002505A5B1C23400000000000000463C0000000000000000558F56"
"F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB7"
"2AED0000005161400000000007C02982140B639A808E3B9742A25334E5CF68EEDEDE52F54E83141565EED165BA79999425204A8491C73B1301"
"E34FE1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC4565183948C67127DAA598D"
"1197F3DEDC4552F64034F3B8213060B8903FD6C8A561E62D0000039C62400000000D7F518BE1E7220000000024057A8B032D0000039B202B00"
"000290202C000000082032057A6CBC62400000000D77916281140B639A808E3B9742A25334E5CF68EEDEDE52F54E8914CEAECC5B87EA043BD9"
"8E1B4FE8663AC59D5C3518E1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45658"
"7E28972F3B63D2260C0671E59593EE6C4D27AB857D1AF230F5CAA959BB3EACE624048A8B6F624000001B84AABF6AE1E7220000000024048A8B"
"702D00000000624000001B84AADE5581141565EED165BA79999425204A8491C73B1301E34FE1E1E41100645698929E4419455BBB551BF09C08"
"EEBEE19021E6B7E3D1D989968EF49970F10130E722000000012505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1F"
"D7B40F710AC45898929E4419455BBB551BF09C08EEBEE19021E6B7E3D1D989968EF49970F101305A00080FA03A44668A2B96DFDE11BF0817CC"
"6DF60C4E3508D43EB72AED00000051E1E1E411003756A2A1BC9A62AAEB2A2A70F895587A3FB752514AA03F8C6E7C84864653B8673E03E72200"
"00000125059FB7B734000000000000003C3C000000000000000055172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235B"
"EBBE5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED0000005161400000000007A1208214246B3E06AB367AB9614566"
"B6F90C718B52A4440883141565EED165BA79999425204A8491C73B1301E34FE1E1E511006125059DE0B155C0F457C1104D45881194D53DBB40"
"B81F9D812FC507637C4294527C06FDD69A3856A969AD4B14312DFA8739258A6A94C0868F1A3E7D1A1EF4839983521A4F6953FCE66240000000"
"0011B75AE1E7220000000024046044C82D00000000202B00001836202C00000F3F62400000000012057A81143A44668A2B96DFDE11BF0817CC"
"6DF60C4E3508D48914651B85AF14BE4F60AFBF5ADAA4F367061C2DA1D4E1E1E51100642505A5B1C2558F56F16F29177518F3DD6CF827085D7B"
"9E2806CD5EBDE810DD1FD7B40F710AC456CE29B8547FC486F45704C1DE539B748587872AA803FB53FB938E282547DF2CFEE722000000003200"
"0000000000004558DE4F51B35BE5A98D0C97BE07378E9CB56FFE3F861E544E0ABAC5ED765E2F781982140B639A808E3B9742A25334E5CF68EE"
"DEDE52F54EE1E1E51100642505A4EA55556022061889BB4DA5242B7AB048C3D751ACEE5788ED4120B2E70F68E9A322A1E956D7A8E1C70CD8A0"
"3AE9BAA41BC0898471FC26DA3712748A803B2F32007CDCB0DCE7220000000031000000000000003D32000000000000003B58B6629B8F178A18"
"C2926F1ADA669262B5D362BFC2DB1417D837CBF382AA37F2D88214246B3E06AB367AB9614566B6F90C718B52A44408E1E1F1031000";
constinit auto const kHASH2 = "D7604B124D5D9C89EC1854A6CBD5A1FFD92502E945411B9C8DE397E7F19A74F8";
auto
createTestData()
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN, kHASH2, kTXN_META2, kTXN_HEX2),
util::createTransaction(ripple::TxType::ttAMM_CREATE), // not NFT - will be filtered
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), // not unique - will be filtered
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = {},
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
} // namespace
struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
etlng::impl::NFTExt ext_{backend_};
};
TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
});
EXPECT_CALL(*backend_, writeNFTTransactions);
ext_.onLedgerData(data);
}
TEST_F(NFTExtTests, OnInitialDataFiltersAndWritesNFTs)
{
auto const data = createTestData();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
});
EXPECT_CALL(*backend_, writeNFTTransactions);
ext_.onInitialData(data);
}
TEST_F(NFTExtTests, OnInitialObjectExtractsAndWritesNFTData)
{
auto const data = util::createObjectWithTwoNFTs();
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { EXPECT_EQ(nfts.size(), 2); });
ext_.onInitialObject(kSEQ, data);
}

View File

@@ -0,0 +1,639 @@
//------------------------------------------------------------------------------
/*
This file is part of clio: https://github.com/XRPLF/clio
Copyright (c) 2025, the clio developers.
Permission to use, copy, modify, and distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
//==============================================================================
#include "data/DBHelpers.hpp"
#include "data/Types.hpp"
#include "etlng/Models.hpp"
#include "etlng/impl/ext/Successor.hpp"
#include "util/Assert.hpp"
#include "util/BinaryTestObject.hpp"
#include "util/MockAssert.hpp"
#include "util/MockBackendTestFixture.hpp"
#include "util/MockLedgerCache.hpp"
#include "util/MockPrometheus.hpp"
#include "util/StringUtils.hpp"
#include "util/TestObject.hpp"
#include <gmock/gmock.h>
#include <gtest/gtest.h>
#include <xrpl/basics/base_uint.h>
#include <xrpl/protocol/TxFormats.h>
#include <algorithm>
#include <iterator>
#include <optional>
#include <queue>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
using namespace etlng::impl;
using namespace data;
namespace {
constinit auto const kSEQ = 123u;
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
auto
createTestData(std::vector<etlng::model::Object> objects)
{
auto transactions = std::vector{
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
};
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
return etlng::model::LedgerData{
.transactions = std::move(transactions),
.objects = std::move(objects),
.successors = {},
.edgeKeys = {},
.header = header,
.rawHeader = {},
.seq = kSEQ
};
}
[[maybe_unused]] auto
createInitialTestData(std::vector<ripple::uint256> edgeKeys)
{
// initial data expects objects to be empty as well as non-empty edgeKeys
ASSERT(not edgeKeys.empty(), "Initial data requires edgeKeys");
auto ret = createTestData({});
ret.edgeKeys = std::make_optional<std::vector<std::string>>();
std::ranges::transform(edgeKeys, std::back_inserter(ret.edgeKeys.value()), &uint256ToString);
return ret;
}
} // namespace
struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest {
protected:
MockLedgerCache cache_;
etlng::impl::SuccessorExt ext_{backend_, cache_};
};
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent)
{
auto const data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(false));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
}
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffersAndSuccessorsNotPresent)
{
auto const data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ - 1));
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(Blob{'0'}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillRepeatedly(testing::Return(LedgerObject{}));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
[[maybe_unused]] testing::InSequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(LedgerObject{.key = createdObj.key, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(deletedObj.key);
auto const oldCachedObj = createdObj.data;
[[maybe_unused]] testing::InSequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(data::kLAST_KEY)));
ext_.onLedgerData(data);
}
TEST_F(
SuccessorExtTests,
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(deletedObj.key);
auto const oldCachedObj = createdObj.data;
[[maybe_unused]] testing::InSequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(LedgerObject{.key = deletedObj.key, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(deletedObj.key)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const predKey =
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"
));
auto const succKey =
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
));
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ))
.WillOnce(testing::Return(data::LedgerObject{.key = predKey, .blob = {}}));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ))
.WillOnce(testing::Return(data::LedgerObject{.key = succKey, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(predKey), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(succKey)));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
auto data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const succ = util::createSuccessor();
data.successors = {succ, succ, succ};
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, auto{succ.firstBook}))
.Times(data.successors->size());
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.predecessor}, kSEQ, auto{createdObj.keyRaw}));
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.keyRaw}, kSEQ, auto{createdObj.successor}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
auto succ = util::createSuccessor();
succ.firstBook = {}; // empty will be transformed into kLAST_KEY
data.successors = {succ, succ};
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, uint256ToString(data::kLAST_KEY)))
.Times(data.successors->size());
EXPECT_CALL(*backend_, writeSuccessor(auto{deletedObj.predecessor}, kSEQ, auto{deletedObj.successor}));
ext_.onLedgerData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
[[maybe_unused]] testing::Sequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = {'0'}};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(std::nullopt));
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
[[maybe_unused]] testing::Sequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = {'0'}};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {}}));
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey)));
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto const data = createInitialTestData({firstKey, secondKey});
auto successorChain = std::queue<ripple::uint256>();
successorChain.push(firstKey);
successorChain.push(secondKey);
auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created);
auto const bookBase = getBookBase(bookBaseObj.key);
[[maybe_unused]] testing::Sequence const inSeq;
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
.Times(3)
.InSequence(inSeq)
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
if (successorChain.empty())
return std::nullopt;
auto v = successorChain.front();
successorChain.pop();
return data::LedgerObject{.key = v, .blob = bookBaseObj.data};
});
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(bookBase, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
.WillRepeatedly(testing::Return(data::LedgerObject{.key = firstKey, .blob = data::Blob{'1'}}));
EXPECT_CALL(
*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_)
); // Called once because firstKey returned repeatedly above
for (auto const& key : data.edgeKeys.value()) {
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
.InSequence(inSeq)
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {'1'}}));
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))).InSequence(inSeq);
}
ext_.onInitialData(data);
}
TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey)
{
using namespace etlng::model;
auto const lastKey = std::string{};
auto const data = std::vector{
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
),
};
std::string lk = lastKey;
for (auto const& obj : data) {
if (not lk.empty())
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
lk = uint256ToString(obj.key);
}
ext_.onInitialObjects(kSEQ, data, lastKey);
}
TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey)
{
using namespace etlng::model;
auto const lastKey =
uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"));
auto const data = std::vector{
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
),
util::createObject(
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
),
};
std::string lk = lastKey;
for (auto const& obj : data) {
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
lk = uint256ToString(obj.key);
}
ext_.onInitialObjects(kSEQ, data, lastKey);
}
struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests {};
TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
auto const data = createTestData({
deletedObj,
util::createObject(Object::ModType::Modified),
});
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
}
TEST_F(
SuccessorExtAssertTests,
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache
)
{
using namespace etlng::model;
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
auto const data = createTestData({
createdObj,
util::createObject(Object::ModType::Modified),
});
auto const bookBase = getBookBase(createdObj.key);
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull)
{
using namespace etlng::model;
auto const data = createTestData({
util::createObject(Object::ModType::Modified),
util::createObject(Object::ModType::Created),
});
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(false));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys)
{
using namespace etlng::model;
auto data = createTestData({});
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects)
{
using namespace etlng::model;
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
auto data = createInitialTestData({firstKey, secondKey});
data.objects = {util::createObject()};
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
}

View File

@@ -419,7 +419,8 @@ TEST_F(RPCHelpersTest, DeliverMaxAliasV1)
for (size_t i = 0; i < inputArray.size(); i++) {
auto req = boost::json::parse(inputArray[i]).as_object();
insertDeliverMaxAlias(req, 1);
EXPECT_EQ(req, boost::json::parse(outputArray[i]).as_object());
auto const expectedReq = boost::json::parse(outputArray[i]).as_object();
EXPECT_EQ(req, expectedReq) << req << "\n" << expectedReq;
}
}

View File

@@ -41,9 +41,9 @@
using namespace rpc;
using namespace data;
using namespace testing;
namespace json = boost::json;
using namespace testing;
namespace {

View File

@@ -522,9 +522,7 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest)
"limit": "100",
"limit_peer": "200",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
},
{
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
@@ -533,9 +531,7 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest)
"limit": "200",
"limit_peer": "100",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
}
]
})";
@@ -737,7 +733,6 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField)
"limit_peer": "200",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": true,
"peer_authorized": true,
"freeze_peer": true,
@@ -752,7 +747,6 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField)
"quality_in": 0,
"quality_out": 0,
"no_ripple": true,
"no_ripple_peer": false,
"authorized": true,
"freeze": true,
"deep_freeze": true
@@ -992,9 +986,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin)
"limit": "100",
"limit_peer": "200",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
}},
{{
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
@@ -1003,9 +995,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin)
"limit": "200",
"limit_peer": "100",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
}}
]
}})",
@@ -1073,9 +1063,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax)
"limit": "100",
"limit_peer": "200",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
}},
{{
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
@@ -1084,9 +1072,7 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax)
"limit": "200",
"limit_peer": "100",
"quality_in": 0,
"quality_out": 0,
"no_ripple": false,
"no_ripple_peer": false
"quality_out": 0
}}
]
}})",

View File

@@ -31,7 +31,6 @@
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/context.hpp>
#include <boost/beast/core/flat_buffer.hpp>
#include <boost/beast/http/field.hpp>
#include <boost/beast/http/message.hpp>
@@ -300,8 +299,7 @@ TEST_F(HttpConnectionTests, Upgrade)
[&]() { ASSERT_TRUE(expectedResult.has_value()) << expectedResult.error().message(); }();
[&]() { ASSERT_TRUE(expectedResult.value()); }();
std::optional<boost::asio::ssl::context> sslContext;
auto expectedWsConnection = connection.upgrade(sslContext, tagDecoratorFactory_, yield);
auto expectedWsConnection = connection.upgrade(tagDecoratorFactory_, yield);
[&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }();
});
}

View File

@@ -36,7 +36,6 @@
#include <boost/asio/io_context.hpp>
#include <boost/asio/post.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/asio/ssl/context.hpp>
#include <boost/asio/steady_timer.hpp>
#include <boost/beast/core/flat_buffer.hpp>
#include <boost/beast/http/status.hpp>
@@ -74,8 +73,7 @@ struct WebWsConnectionTests : SyncAsioContextTest {
ASSERT_TRUE(expectedTrue.value()) << "Expected upgrade request";
}();
std::optional<boost::asio::ssl::context> sslContext;
auto expectedWsConnection = httpConnection.upgrade(sslContext, tagDecoratorFactory_, yield);
auto expectedWsConnection = httpConnection.upgrade(tagDecoratorFactory_, yield);
[&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }();
auto connection = std::move(expectedWsConnection).value();
auto wsConnectionPtr = dynamic_cast<PlainWsConnection*>(connection.release());

View File

@@ -6,6 +6,7 @@ set(GO_SOURCE_DIR "${CMAKE_SOURCE_DIR}/tools/snapshot")
set(PROTO_INC_DIR "${xrpl_PACKAGE_FOLDER_RELEASE}/include/xrpl/proto")
set(PROTO_SOURCE_DIR "${PROTO_INC_DIR}/org/xrpl/rpc/v1/")
set(GO_OUTPUT "${CMAKE_BINARY_DIR}/clio_snapshot")
file(GLOB_RECURSE GO_SOURCES ${GO_SOURCE_DIR}/*.go)
set(PROTO_FILES
${PROTO_SOURCE_DIR}/xrp_ledger.proto ${PROTO_SOURCE_DIR}/ledger.proto ${PROTO_SOURCE_DIR}/get_ledger.proto
@@ -34,7 +35,7 @@ endforeach()
foreach (proto ${PROTO_FILES})
get_filename_component(proto_name ${proto} NAME_WE)
add_custom_command(
OUTPUT ${GO_SOURCE_DIR}/${proto_name}.pb.go
OUTPUT ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go
COMMAND
protoc ${GO_OPTS} ${GRPC_OPTS}
--go-grpc_out=${GO_SOURCE_DIR} -I${PROTO_INC_DIR} ${proto} --plugin=${GOPATH_VALUE}/bin/protoc-gen-go
@@ -44,7 +45,7 @@ foreach (proto ${PROTO_FILES})
VERBATIM
)
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${proto_name}.pb.go)
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go)
endforeach ()
add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
@@ -52,15 +53,16 @@ add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
add_custom_target(run_go_tests
COMMAND go test ./...
WORKING_DIRECTORY ${GO_SOURCE_DIR}
DEPENDS ${GENERATED_GO_FILES}
COMMENT "Running clio_snapshot unittests"
VERBATIM
DEPENDS ${GENERATED_GO_FILES}
)
add_custom_command(
OUTPUT ${GO_OUTPUT}
COMMAND ${GO_EXECUTABLE} build -o ${GO_OUTPUT} ${GO_SOURCE_DIR}
WORKING_DIRECTORY ${GO_SOURCE_DIR}
DEPENDS ${GO_SOURCES}
COMMENT "Building clio_snapshot"
VERBATIM
)

View File

@@ -6,6 +6,7 @@ toolchain go1.22.11
require (
github.com/golang/mock v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/spf13/pflag v1.0.6
github.com/stretchr/testify v1.10.0
google.golang.org/grpc v1.69.4

View File

@@ -12,6 +12,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=

View File

@@ -4,20 +4,20 @@ import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"sync"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
"xrplf/clio/clio_snapshot/internal/ledgers"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/protobuf/proto"
)
const (
deltaDataFolderDiv = 10000
grpcUser = "clio-snapshot"
markerNum = 16
grpcUser = "clio-snapshot"
markerNum = 16
maxConcurrency = 256
firstAvailableLedger = 32570
)
type gRPCClient struct {
@@ -44,7 +44,25 @@ func createGRPCClient(serverAddr string) (*gRPCClient, error) {
}, nil
}
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
func getLedgerDeltaDataInParallel(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, ledgersHouse *ledgers.LedgersHouse) {
sem := make(chan struct{}, maxConcurrency)
var wg sync.WaitGroup
for i := startSeq; i <= endSeq; i++ {
wg.Add(1)
sem <- struct{}{}
go func(seq uint32) {
defer wg.Done()
log.Printf("Process delta sequence: %d\n", seq)
getLedgerDeltaData(client, seq, ledgersHouse)
<-sem
}(i)
}
wg.Wait()
}
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgersHouse *ledgers.LedgersHouse) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -56,49 +74,27 @@ func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path st
}
request.Ledger = ledger
request.User = grpcUser
request.GetObjectNeighbors = true
request.Transactions = true
request.Expand = true
request.GetObjects = true
// The first available ledger doesn't have diff data
request.GetObjects = firstAvailableLedger != seq
request.GetObjectNeighbors = firstAvailableLedger != seq
response, err := client.GetLedger(ctx, &request)
if err != nil {
log.Fatalf("Error getting ledger data: %v", err)
log.Fatalf("Error getting ledger delta data: %v - seq: %d", err, seq)
}
saveLedgerDeltaData(seq, response, path)
err = ledgersHouse.WriteLedgerDeltaData(seq, response)
if err != nil {
log.Fatalf("Error writing ledger delta data: %v", err)
}
log.Printf("Processing delta sequence: %d\n", seq)
}
func roundDown(n uint32, roundTo uint32) uint32 {
if roundTo == 0 {
return n
}
return n - (n % roundTo)
}
func saveLedgerDeltaData(seq uint32, response *pb.GetLedgerResponse, path string) {
subPath := filepath.Join(path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
err := os.MkdirAll(subPath, os.ModePerm)
if err != nil {
log.Fatalf("Error creating directory: %v", err)
}
protoData, err := proto.Marshal(response)
if err != nil {
log.Fatalf("Error marshalling data: %v", err)
}
filePath := filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
err = os.WriteFile(filePath, protoData, 0644)
if err != nil {
log.Fatalf("failed to write file: %v", err)
}
}
func generateMarkers(markerNum uint32) [][32]byte {
var byteArray [32]byte
@@ -114,19 +110,7 @@ func generateMarkers(markerNum uint32) [][32]byte {
return byteArrayList
}
func saveLedgerData(path string, data *pb.GetLedgerDataResponse) {
protoData, err := proto.Marshal(data)
if err != nil {
log.Fatalf("Error marshalling data: %v", err)
}
err = os.WriteFile(path, protoData, 0644)
if err != nil {
log.Fatalf("failed to write file: %v", err)
}
}
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, path string) {
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, ledgerHouse *ledgers.LedgersHouse) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -143,25 +127,22 @@ func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byt
}
request.User = grpcUser
subPath := filepath.Join(path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%x", marker))
err := os.MkdirAll(subPath, os.ModePerm)
if err != nil {
log.Fatalf("Error creating directory: %v", err)
}
for request.Marker != nil {
res, err := client.GetLedgerData(ctx, &request)
if err != nil {
log.Fatalf("Error getting ledger data: %v", err)
}
filePath := filepath.Join(subPath, fmt.Sprintf("%x.dat", request.Marker))
saveLedgerData(filePath, res)
err = ledgerHouse.WriteLedgerData(seq, request.Marker, res)
if err != nil {
log.Fatalf("Error writing ledger data: %v", err)
}
log.Printf("Saving ledger data %x", request.Marker)
request.Marker = res.Marker
}
}
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHouse *ledgers.LedgersHouse) {
log.Printf("Processing full sequence: %d\n", seq)
markers := generateMarkers(markerNum)
@@ -176,11 +157,9 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
end = markers[i+1][:]
}
fmt.Printf("Got ledger data marker: %x-%x\n", marker, end)
go func() {
defer wg.Done()
getLedgerData(client, seq, marker[:], end, path)
getLedgerData(client, seq, marker[:], end, ledgerHouse)
}()
}
@@ -188,19 +167,7 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path str
wg.Wait()
}
func checkPath(path string) {
if _, err := os.Stat(path); os.IsNotExist(err) {
// Create the directory if it doesn't exist
err := os.MkdirAll(path, os.ModePerm)
if err != nil {
log.Fatalf("Error creating directory: %v", err)
}
}
}
func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
checkPath(path)
client, err := createGRPCClient(grpcServer)
if err != nil {
log.Fatalf("Error creating gRPC client: %v", err)
@@ -212,20 +179,21 @@ func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, pat
}
func exportFromFullLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
ledgersHouse := ledgers.NewLedgersHouse(path)
getLedgerFullData(client, startSeq, path)
getLedgerFullData(client, startSeq, ledgersHouse)
//We need to fetch the ledger header and txs for startSeq as well
for i := startSeq; i <= endSeq; i++ {
getLedgerDeltaData(client, i, path)
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
err := ledgersHouse.SetRange(startSeq, endSeq)
if err != nil {
log.Fatalf("Error writing range: %v", err)
}
log.Printf("Exporting from full ledger: %d to %d at path %s\n", startSeq, endSeq, path)
}
func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
checkPath(path)
client, err := createGRPCClient(grpcServer)
if err != nil {
log.Fatalf("Error creating gRPC client: %v", err)
@@ -237,8 +205,27 @@ func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, pa
}
func exportFromDeltaLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
for i := startSeq; i <= endSeq; i++ {
getLedgerDeltaData(client, i, path)
ledgersHouse := ledgers.NewLedgersHouse(path)
_, oldEnd, err := ledgersHouse.GetRange()
if err != nil {
log.Fatalf("Can't find existing snapshot to extend: %v", err)
}
if oldEnd < startSeq-1 {
log.Fatalf("Missing delta ledger from %d to %d", oldEnd, startSeq)
}
if oldEnd >= endSeq {
log.Fatalf("The snapshot already contains the requested delta ledger")
}
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
err = ledgersHouse.AppendDeltaLedger(startSeq, endSeq)
if err != nil {
log.Fatalf("Error writing new range: %v", err)
}
log.Printf("Exporting from ledger: %d to %d at path %s\n", startSeq, endSeq, path)

View File

@@ -1,26 +1,48 @@
package export
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/assert"
"xrplf/clio/clio_snapshot/internal/ledgers"
"xrplf/clio/clio_snapshot/mocks"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
"github.com/golang/mock/gomock"
)
// Matcher used to verify the GetLedgerRequest parameters
type LedgerRequestMatcher struct {
expectedObjects bool
expectedNeighbors bool
}
func (m LedgerRequestMatcher) Matches(x interface{}) bool {
req, ok := x.(*pb.GetLedgerRequest)
return ok && req.GetObjects == m.expectedObjects && req.GetObjectNeighbors == m.expectedNeighbors
}
func (m LedgerRequestMatcher) String() string {
return fmt.Sprintf("LedgerRequest with objects=%v neighbors=%v", m.expectedObjects, m.expectedNeighbors)
}
func matchObjectsEquals(objects bool, neighbors bool) gomock.Matcher {
return LedgerRequestMatcher{objects, neighbors}
}
func TestExportDeltaLedgerData(t *testing.T) {
tests := []struct {
name string
startSeq uint32
endSeq uint32
}{
{"OneSeq", 1, 1},
{"MultipleSeq", 1, 20},
{"EndSeqLessThanStartSeq", 20, 1},
{"OneSeq", 700000, 700000},
{"MultipleSeq", 700000, 700019},
{"FirstAvailableLedger", firstAvailableLedger, firstAvailableLedger},
{"FirstAvailableLedgerMultipleSeq", firstAvailableLedger, firstAvailableLedger + 2},
}
for _, tt := range tests {
@@ -37,15 +59,26 @@ func TestExportDeltaLedgerData(t *testing.T) {
times = 0
}
mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(int(times))
if tt.startSeq == firstAvailableLedger {
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(false, false)).Return(mockResponse, nil).Times(1)
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times) - 1)
} else {
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times))
}
os.MkdirAll("test", os.ModePerm)
manifest := ledgers.NewManifest("test")
manifest.SetLedgerRange(tt.startSeq-1, tt.startSeq-1)
defer os.RemoveAll("test")
exportFromDeltaLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
_, err := os.Stat("test")
assert.Equal(t, os.IsNotExist(err), tt.endSeq < tt.startSeq)
seq1, seq2, err := manifest.Read()
assert.Nil(t, err)
assert.Equal(t, tt.startSeq-1, seq1)
assert.Equal(t, tt.endSeq, seq2)
})
}
}
@@ -92,26 +125,6 @@ func TestExportFullLedgerData(t *testing.T) {
}
}
func TestRoundDown(t *testing.T) {
tests := []struct {
name string
in1 uint32
in2 uint32
out uint32
}{
{"RoundDownToZero", 10, 0, 10},
{"RoundDown12To10", 12, 10, 10},
{"RoundDownToOne", 13, 1, 13},
{"RoundDown100", 103, 100, 100},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
})
}
}
func TestGenerateMarkers(t *testing.T) {
tests := []struct {
name string
@@ -132,22 +145,3 @@ func TestGenerateMarkers(t *testing.T) {
})
}
}
func TestCheckPath(t *testing.T) {
tests := []struct {
name string
path string
}{
{"Path", "test"},
{"NestedPath", "test/test"}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
checkPath(tt.path)
defer os.RemoveAll(tt.path)
_, err := os.Stat(tt.path)
assert.False(t, os.IsNotExist(err))
})
}
}

View File

@@ -0,0 +1,133 @@
package ledgers
import (
"fmt"
"os"
"path/filepath"
"google.golang.org/protobuf/proto"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
)
const deltaDataFolderDiv = 10000
const readWritePerm = 0644
func convertInnerMarkerToMarker(in []byte) []byte {
if in == nil {
return nil
}
out := make([]byte, len(in))
out[0] = in[0] & 0xf0
return out
}
func checkPath(path string) error {
dir := filepath.Dir(path)
if _, err := os.Stat(dir); os.IsNotExist(err) {
// Create the directory if it doesn't exist
err := os.MkdirAll(dir, os.ModePerm)
if err != nil {
return fmt.Errorf("Error creating directory: %s,%v", path, err)
}
}
return nil
}
func roundDown(n uint32, roundTo uint32) uint32 {
if roundTo == 0 {
return n
}
return n - (n % roundTo)
}
type LedgersHouse struct {
path string
manifest *Manifest
}
func NewLedgersHouse(path string) *LedgersHouse {
return &LedgersHouse{path: path, manifest: NewManifest(path)}
}
func (lh *LedgersHouse) deltaDataPath(seq uint32) string {
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
return filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
}
func (lh *LedgersHouse) fullDataPath(seq uint32, marker string, innerMarker string) string {
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%s", marker))
return filepath.Join(subPath, fmt.Sprintf("%s.dat", innerMarker))
}
func (lh *LedgersHouse) ReadLedgerDeltaData(seq uint32) (*pb.GetLedgerResponse, error) {
path := lh.deltaDataPath(seq)
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var ledger pb.GetLedgerResponse
if err := proto.Unmarshal(data, &ledger); err != nil {
return nil, err
}
return &ledger, nil
}
func (lh *LedgersHouse) WriteLedgerDeltaData(seq uint32, data *pb.GetLedgerResponse) error {
path := lh.deltaDataPath(seq)
err := checkPath(path)
if err != nil {
return err
}
dataBytes, err := proto.Marshal(data)
if err != nil {
return err
}
return os.WriteFile(path, dataBytes, readWritePerm)
}
func (lh *LedgersHouse) ReadLedgerData(seq uint32, innerMarker []byte) (*pb.GetLedgerDataResponse, error) {
marker := convertInnerMarkerToMarker(innerMarker)
path := lh.fullDataPath(seq, fmt.Sprintf("%x", marker), fmt.Sprintf("%x", innerMarker))
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var ledger pb.GetLedgerDataResponse
if err := proto.Unmarshal(data, &ledger); err != nil {
return nil, err
}
return &ledger, nil
}
func (lh *LedgersHouse) WriteLedgerData(seq uint32, innerMarker []byte, data *pb.GetLedgerDataResponse) error {
path := lh.fullDataPath(seq, fmt.Sprintf("%x", convertInnerMarkerToMarker(innerMarker)), fmt.Sprintf("%x", innerMarker))
err := checkPath(path)
if err != nil {
return err
}
dataBytes, err := proto.Marshal(data)
if err != nil {
return err
}
return os.WriteFile(path, dataBytes, readWritePerm)
}
func (lh *LedgersHouse) SetRange(startSeq uint32, endSeq uint32) error {
return lh.manifest.SetLedgerRange(startSeq, endSeq)
}
func (lh *LedgersHouse) AppendDeltaLedger(startSeq uint32, endSeq uint32) error {
return lh.manifest.AppendDeltaLedger(startSeq, endSeq)
}
func (lh *LedgersHouse) IsExist() bool {
return lh.manifest.IsExist()
}
func (lh *LedgersHouse) GetRange() (uint32, uint32, error) {
return lh.manifest.Read()
}

View File

@@ -0,0 +1,188 @@
package ledgers
import (
"os"
"path/filepath"
"testing"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
"github.com/stretchr/testify/assert"
)
func TestCheckPath(t *testing.T) {
tests := []struct {
name string
path string
}{
{"Path", "test/d.dat"},
{"NestedPath", "test/test/d.dat"}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := checkPath(tt.path)
assert.NoError(t, err)
dir := filepath.Dir(tt.path)
defer os.RemoveAll("test")
_, err = os.Stat(dir)
assert.False(t, os.IsNotExist(err))
})
}
}
func TestRoundDown(t *testing.T) {
tests := []struct {
name string
in1 uint32
in2 uint32
out uint32
}{
{"RoundDownToZero", 10, 0, 10},
{"RoundDown12To10", 12, 10, 10},
{"RoundDownToOne", 13, 1, 13},
{"RoundDown100", 103, 100, 100},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
})
}
}
func TestConvertInnerMarkerToMarker(t *testing.T) {
tests := []struct {
name string
in []byte
out []byte
}{
{"SingleByte", []byte{0x01}, []byte{0x00}},
{"MultipleBytes", []byte{0x01, 0x02, 0x03}, []byte{0x00, 0x00, 0x00}},
{"MultipleBytes2", []byte{0xf1, 0x02, 0x03}, []byte{0xf0, 0x00, 0x00}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equal(t, convertInnerMarkerToMarker(tt.in), tt.out)
})
}
}
func TestLedgersHouseGetDeltaPath(t *testing.T) {
lh := NewLedgersHouse("testdata")
assert.Equal(t, lh.deltaDataPath(12345), "testdata/ledger_diff_10000/12345.dat")
assert.Equal(t, lh.deltaDataPath(3), "testdata/ledger_diff_0/3.dat")
assert.Equal(t, lh.deltaDataPath(0), "testdata/ledger_diff_0/0.dat")
}
func TestLedgersHouseGetFullDataPath(t *testing.T) {
lh := NewLedgersHouse("testdata")
assert.Equal(t, lh.fullDataPath(12345, "fffff", "ababab"), "testdata/ledger_data_12345/marker_fffff/ababab.dat")
}
func TestLedgerHouseLedgerDeltaData(t *testing.T) {
defer os.RemoveAll("testdata")
lh := NewLedgersHouse("testdata")
data, err := lh.ReadLedgerDeltaData(12345)
assert.True(t, data == nil)
assert.True(t, err != nil)
lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
data, err = lh.ReadLedgerDeltaData(12345)
assert.True(t, data != nil)
assert.True(t, err == nil)
}
func TestLedgerHouseInvalidLedgerDeltaPath(t *testing.T) {
lh := NewLedgersHouse("/etc")
data, err := lh.ReadLedgerDeltaData(12345)
assert.True(t, data == nil)
assert.True(t, err != nil)
err = lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
assert.True(t, err != nil)
}
func TestLedgerHouseLedgerData(t *testing.T) {
defer os.RemoveAll("testdata")
lh := NewLedgersHouse("testdata")
data, err := lh.ReadLedgerData(12345, []byte{0x01})
assert.True(t, data == nil)
assert.True(t, err != nil)
lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
data, err = lh.ReadLedgerData(12345, []byte{0x01})
assert.True(t, data != nil)
assert.True(t, err == nil)
}
func TestLedgerHouseInvalidLedgerDataPath(t *testing.T) {
lh := NewLedgersHouse("/etc")
data, err := lh.ReadLedgerData(12345, []byte{0x01})
assert.True(t, data == nil)
assert.True(t, err != nil)
err = lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
assert.True(t, err != nil)
}
func TestLedgersHouseManifest(t *testing.T) {
defer os.RemoveAll("testdata")
lh := NewLedgersHouse("testdata")
startSeq, endSeq, err := lh.GetRange()
assert.True(t, err != nil)
assert.Equal(t, startSeq, uint32(0))
assert.Equal(t, endSeq, uint32(0))
assert.False(t, lh.IsExist())
lh.SetRange(1, 100)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(100))
lh.AppendDeltaLedger(100, 200)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(200))
lh.AppendDeltaLedger(201, 300)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(300))
err = lh.AppendDeltaLedger(201, 100)
assert.True(t, err != nil)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(300))
err = lh.AppendDeltaLedger(302, 350)
assert.True(t, err != nil)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(300))
err = lh.AppendDeltaLedger(0, 350)
assert.True(t, err != nil)
assert.True(t, lh.IsExist())
startSeq, endSeq, err = lh.GetRange()
assert.True(t, err == nil)
assert.Equal(t, startSeq, uint32(1))
assert.Equal(t, endSeq, uint32(300))
}

View File

@@ -0,0 +1,91 @@
package ledgers
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
)
const (
fileName = "manifest.txt"
)
type Manifest struct {
folderPath string
filePath string
}
func NewManifest(folderPath string) *Manifest {
return &Manifest{
folderPath: folderPath,
filePath: filepath.Join(folderPath, fileName),
}
}
func (fm *Manifest) SetLedgerRange(start uint32, end uint32) error {
content := fmt.Sprintf("%d|%d", start, end)
return fm.writeToFile(content)
}
func (fm *Manifest) AppendDeltaLedger(delta1 uint32, delta2 uint32) error {
start, end, err := fm.Read()
if err != nil {
return err
}
//rewrite the range if new delta can extend the current range continuously
if delta1 >= start && (end+1) >= delta1 && delta2 >= delta1 {
return fm.SetLedgerRange(start, delta2)
}
return fmt.Errorf("Invalid delta ledger range")
}
func (fm *Manifest) writeToFile(content string) error {
os.MkdirAll(fm.folderPath, os.ModePerm)
file, err := os.OpenFile(fm.filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
return err
}
defer file.Close()
_, err = file.WriteString(content)
if err != nil {
return err
}
return nil
}
func (fm *Manifest) IsExist() bool {
_, err := os.Stat(fm.filePath)
return !os.IsNotExist(err)
}
func (fm *Manifest) Read() (uint32, uint32, error) {
content, err := os.ReadFile(fm.filePath)
if err != nil {
return 0, 0, err
}
if len(content) == 0 {
return 0, 0, nil
}
parts := strings.Split(string(content), "|")
if len(parts) != 2 {
return 0, 0, fmt.Errorf("file content is not in expected format")
}
part1, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("error parsing the first part: %v", err)
}
part2, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 32)
if err != nil {
return 0, 0, fmt.Errorf("error parsing the second part: %v", err)
}
return uint32(part1), uint32(part2), nil
}

View File

@@ -0,0 +1,41 @@
package ledgers
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestManifest(t *testing.T) {
manifest := NewManifest("testdata")
defer os.RemoveAll("testdata")
assert.False(t, manifest.IsExist())
_, _, err := manifest.Read()
assert.Error(t, err)
err = manifest.SetLedgerRange(1, 10)
assert.NoError(t, err)
err = manifest.AppendDeltaLedger(11, 20)
assert.NoError(t, err)
assert.True(t, manifest.IsExist())
err = manifest.AppendDeltaLedger(22, 30)
assert.Error(t, err)
start, end, err := manifest.Read()
assert.NoError(t, err)
assert.Equal(t, start, uint32(1))
assert.Equal(t, end, uint32(20))
}
func TestManifestInvalidPath(t *testing.T) {
manifest := NewManifest("/")
assert.False(t, manifest.IsExist())
_, _, err := manifest.Read()
assert.Error(t, err)
err = manifest.SetLedgerRange(1, 10)
assert.Error(t, err)
}

View File

@@ -13,9 +13,9 @@ type Args struct {
EndSeq uint32
Path string
GrpcServer string
WsServer string
ServerMode bool
GrpcPort uint32
WsPort uint32
ShowRange bool
}
func Parse() (*Args, error) {
@@ -27,10 +27,10 @@ func Parse() (*Args, error) {
seq := fs.Uint32("start_seq", 0, "Starting sequence number")
endSeq := fs.Uint32("end_seq", 0, "Ending sequence number")
path := fs.StringP("path", "p", "", "Path to the data")
grpcServer := fs.StringP("grpc_server", "g", "localhost:50051", "rippled's gRPC server address")
grpcServer := fs.StringP("grpc_server", "g", "0.0.0.0:50051", "rippled's gRPC server address")
wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address")
serverMode := fs.BoolP("server", "s", false, "Start server mode")
grpcPort := fs.Uint32("grpc_port", 0, "Port for gRPC server to listen on")
wsPort := fs.Uint32("ws_port", 0, "Port for WebSocket server to listen on")
showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot")
fs.Parse(os.Args[1:])
if *serverMode && *exportMode != "" {
@@ -38,22 +38,26 @@ func Parse() (*Args, error) {
}
if *serverMode {
if *grpcPort == 0 || *wsPort == 0 || *path == "" {
return nil, fmt.Errorf("Invalid usage: --grpc_port and --ws_port and --path are required for server mode.")
if *grpcServer == "" || *wsServer == "" || *path == "" {
return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.")
}
} else if *exportMode != "" {
if *exportMode == "full" || *exportMode == "delta" {
if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.")
}
} else {
return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
}
} else if *showRange {
if *path == "" {
return nil, fmt.Errorf("Invalid usage: --path is required for show range.")
}
} else {
return nil, fmt.Errorf("Invalid usage: --export or --server flag is required.")
return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.")
}
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *serverMode, *grpcPort, *wsPort}, nil
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil
}
func PrintUsage() {

View File

@@ -25,6 +25,7 @@ func TestParse(t *testing.T) {
EndSeq: 10,
Path: "/data",
GrpcServer: "localhost:50051",
WsServer: "0.0.0.0:6006",
ServerMode: false,
},
expectErr: false,
@@ -34,7 +35,7 @@ func TestParse(t *testing.T) {
args: []string{"cmd", "--export=delta", "--start_seq=1"},
want: nil,
expectErr: true,
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.",
},
{
name: "Invalid export mode",
@@ -45,18 +46,24 @@ func TestParse(t *testing.T) {
},
{
name: "Server mode with default grpc server flags",
args: []string{"cmd", "--server", "--ws_port=1234", "--grpc_port=22", "--path=/server_data"},
args: []string{"cmd", "--server", "--path=/server_data"},
want: &Args{
ServerMode: true,
GrpcPort: 22,
WsPort: 1234,
StartSeq: 0,
EndSeq: 0,
Path: "/server_data",
GrpcServer: "localhost:50051",
GrpcServer: "0.0.0.0:50051",
WsServer: "0.0.0.0:6006",
},
expectErr: false,
},
{
name: "Server mode with empty grpc server flag",
args: []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"},
want: nil,
expectErr: true,
errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.",
},
{
name: "Server and export mode together (error)",
args: []string{"cmd", "--server", "--export=full"},
@@ -64,6 +71,24 @@ func TestParse(t *testing.T) {
expectErr: true,
errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
},
{
name: "Show range without path",
args: []string{"cmd", "--range"},
want: nil,
expectErr: true,
errMessage: "Invalid usage: --path is required for show range.",
},
{
name: "Show range",
args: []string{"cmd", "--range", "--path=/range_data"},
want: &Args{
ShowRange: true,
Path: "/range_data",
GrpcServer: "0.0.0.0:50051",
WsServer: "0.0.0.0:6006",
},
expectErr: false,
},
}
for _, tt := range tests {

View File

@@ -0,0 +1,41 @@
package server
import (
"context"
"fmt"
"xrplf/clio/clio_snapshot/internal/ledgers"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
)
// create a server implement the xrpl rpc v1 server interface
type Server struct {
pb.XRPLedgerAPIServiceServer
ledgersHouse *ledgers.LedgersHouse
}
func (s *Server) GetLedger(ctx context.Context, req *pb.GetLedgerRequest) (*pb.GetLedgerResponse, error) {
return s.ledgersHouse.ReadLedgerDeltaData(req.GetLedger().GetSequence())
}
func (s *Server) GetLedgerData(ctx context.Context, req *pb.GetLedgerDataRequest) (*pb.GetLedgerDataResponse, error) {
marker := req.GetMarker()
if marker == nil {
marker = make([]byte, 32)
}
return s.ledgersHouse.ReadLedgerData(req.GetLedger().GetSequence(), marker)
}
func (s *Server) GetLedgerDiff(ctx context.Context, req *pb.GetLedgerDiffRequest) (*pb.GetLedgerDiffResponse, error) {
return nil, fmt.Errorf("GetLedgerDiff not supported")
}
func (s *Server) GetLedgerEntry(ctx context.Context, req *pb.GetLedgerEntryRequest) (*pb.GetLedgerEntryResponse, error) {
return nil, fmt.Errorf("GetLedgerEntry not supported")
}
func newServer(path string) *Server {
s := &Server{}
s.ledgersHouse = ledgers.NewLedgersHouse(path)
return s
}

View File

@@ -0,0 +1,88 @@
package server
import (
"context"
"os"
"testing"
"github.com/stretchr/testify/assert"
"xrplf/clio/clio_snapshot/internal/ledgers"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
)
func TestUnavaibleMethods(t *testing.T) {
srv := newServer("testdata")
req := &pb.GetLedgerDiffRequest{}
_, err := srv.GetLedgerDiff(context.Background(), req)
assert.Error(t, err)
assert.Equal(t, err.Error(), "GetLedgerDiff not supported")
req2 := &pb.GetLedgerEntryRequest{}
_, err = srv.GetLedgerEntry(context.Background(), req2)
assert.Error(t, err)
assert.Equal(t, err.Error(), "GetLedgerEntry not supported")
}
func TestWhenPathIsInvalid(t *testing.T) {
srv := newServer("testdata")
req := &pb.GetLedgerRequest{
Ledger: &pb.LedgerSpecifier{
Ledger: &pb.LedgerSpecifier_Sequence{
Sequence: 2,
},
},
}
_, err := srv.GetLedger(context.Background(), req)
assert.Error(t, err)
assert.Equal(t, err.Error(), "open testdata/ledger_diff_0/2.dat: no such file or directory")
req2 := &pb.GetLedgerDataRequest{
Ledger: &pb.LedgerSpecifier{
Ledger: &pb.LedgerSpecifier_Sequence{
Sequence: 2,
},
},
}
_, err = srv.GetLedgerData(context.Background(), req2)
assert.Error(t, err)
assert.Equal(t, err.Error(), "open testdata/ledger_data_2/marker_0000000000000000000000000000000000000000000000000000000000000000/0000000000000000000000000000000000000000000000000000000000000000.dat: no such file or directory")
}
func TestWhenPathIsValid(t *testing.T) {
srv := newServer("testdata")
ledger := ledgers.NewLedgersHouse("testdata")
defer os.RemoveAll("testdata")
marker := [32]byte{}
ledger.WriteLedgerData(1, marker[:], &pb.GetLedgerDataResponse{})
ledger.WriteLedgerDeltaData(1, &pb.GetLedgerResponse{})
req := &pb.GetLedgerRequest{
Ledger: &pb.LedgerSpecifier{
Ledger: &pb.LedgerSpecifier_Sequence{
Sequence: 1,
},
},
}
res, err := srv.GetLedger(context.Background(), req)
assert.NoError(t, err)
assert.NotNil(t, res)
req2 := &pb.GetLedgerDataRequest{
Ledger: &pb.LedgerSpecifier{
Ledger: &pb.LedgerSpecifier_Sequence{
Sequence: 1,
},
},
}
res2, err := srv.GetLedgerData(context.Background(), req2)
assert.NoError(t, err)
assert.NotNil(t, res2)
}

View File

@@ -0,0 +1,46 @@
package server
import (
"fmt"
"log"
"net"
"xrplf/clio/clio_snapshot/internal/ledgers"
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
"google.golang.org/grpc"
)
func StartServer(grpcServerAddr string, wsServerAddr string, path string) {
ledgersHouse := ledgers.NewLedgersHouse(path)
if !ledgersHouse.IsExist() {
log.Fatalf("Can't start server againist invalid snapshot folder: %s", path)
}
startSeq, endSeq, err := ledgersHouse.GetRange()
if err != nil {
log.Fatalf("Failed to get range: %v", err)
}
lis, err := net.Listen("tcp", grpcServerAddr)
if err != nil {
log.Fatalf("Failed to listen: %v", err)
}
grpcServer := grpc.NewServer()
pb.RegisterXRPLedgerAPIServiceServer(grpcServer, newServer(path))
log.Print("Starting server...")
go grpcServer.Serve(lis)
wsServer := NewWebSocketServer("Snapshot Server", func(message string) string {
//mimic the response of the ledger stream
ledgerStreamReply := fmt.Sprintf("{\"fee_base\":10,\"ledger_hash\":\"A320C67DA7D1250A577AC5AACDF06ADC25E0EEEF7AE5B8D63CE2E1CC7F76A438\",\"ledger_index\":%d,\"ledger_time\":792853443,\"reserve_base\":1000000,\"reserve_inc\":200000,\"txn_count\":0,\"type\":\"ledgerClosed\",\"validated_ledgers\":\"%d-%d\"}",
endSeq, startSeq, endSeq)
return ledgerStreamReply
})
wsServer.Start(wsServerAddr)
select {}
}

View File

@@ -0,0 +1,64 @@
package server
import (
"log"
"net/http"
"github.com/gorilla/websocket"
)
type WebSocketServer struct {
serverName string
callback func(message string) string
upgrader websocket.Upgrader
}
func NewWebSocketServer(serverName string, callback func(message string) string) *WebSocketServer {
return &WebSocketServer{
serverName: serverName,
callback: callback,
upgrader: websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool { return true }, // Allow all connections
},
}
}
func (ws *WebSocketServer) handleConnections() http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
conn, err := ws.upgrader.Upgrade(w, r, nil)
if err != nil {
log.Printf("[%s] Error upgrading to WebSocket: %v", ws.serverName, err)
return
}
defer conn.Close()
log.Printf("[%s] New WebSocket connection established", ws.serverName)
for {
_, msg, err := conn.ReadMessage()
if err != nil {
log.Printf("[%s] Error reading message: %v", ws.serverName, err)
break
}
log.Printf("[%s] Received: %s", ws.serverName, msg)
response := ws.callback(string(msg))
err = conn.WriteMessage(websocket.TextMessage, []byte(response))
log.Printf("[%s] Sending: %s", ws.serverName, response)
if err != nil {
log.Printf("[%s] Error writing message: %v", ws.serverName, err)
break
}
}
}
}
func (ws *WebSocketServer) Start(address string) {
http.HandleFunc("/", ws.handleConnections())
log.Printf("[%s] Starting ws server on address: %s", ws.serverName, address)
err := http.ListenAndServe(address, nil)
if err != nil {
log.Fatalf("[%s] Server failed: %v", ws.serverName, err)
}
}

View File

@@ -4,7 +4,9 @@ import (
"log"
"xrplf/clio/clio_snapshot/internal/export"
"xrplf/clio/clio_snapshot/internal/ledgers"
"xrplf/clio/clio_snapshot/internal/parse_args"
"xrplf/clio/clio_snapshot/internal/server"
)
func main() {
@@ -18,6 +20,18 @@ func main() {
} else if args.ExportMode == "delta" {
export.ExportFromDeltaLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
} else if args.ServerMode {
server.StartServer(args.GrpcServer, args.WsServer, args.Path)
} else if args.ShowRange {
ledgers := ledgers.NewLedgersHouse(args.Path)
if !ledgers.IsExist() {
log.Fatalf("Invalid snapshot folder: %s", args.Path)
}
startSeq, endSeq, err := ledgers.GetRange()
if err == nil {
log.Printf("Snapshot range: %d-%d", startSeq, endSeq)
} else {
log.Fatalf("Failed to get snapshot range: %v", err)
}
}
//TODO: Implement server mode
}