mirror of
https://github.com/XRPLF/clio.git
synced 2025-12-05 16:58:00 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5e7ff66ba6 | ||
|
|
1b1a5e4068 | ||
|
|
1bba437085 | ||
|
|
41fc67748a | ||
|
|
7b043025e8 | ||
|
|
1460d590f1 |
28
.github/actions/prepare_runner/action.yml
vendored
28
.github/actions/prepare_runner/action.yml
vendored
@@ -11,35 +11,9 @@ runs:
|
||||
if: ${{ runner.os == 'macOS' }}
|
||||
shell: bash
|
||||
run: |
|
||||
brew install llvm@14 pkg-config ninja bison ccache jq gh conan@1 ca-certificates
|
||||
brew install llvm@14 pkg-config ninja bison cmake ccache jq gh conan@1 ca-certificates
|
||||
echo "/opt/homebrew/opt/conan@1/bin" >> $GITHUB_PATH
|
||||
|
||||
- name: Install CMake 3.31.6 on mac
|
||||
if: ${{ runner.os == 'macOS' }}
|
||||
shell: bash
|
||||
run: |
|
||||
# Uninstall any existing cmake
|
||||
brew uninstall cmake --ignore-dependencies || true
|
||||
|
||||
# Download specific cmake formula
|
||||
FORMULA_URL="https://raw.githubusercontent.com/Homebrew/homebrew-core/b4e46db74e74a8c1650b38b1da222284ce1ec5ce/Formula/c/cmake.rb"
|
||||
FORMULA_EXPECTED_SHA256="c7ec95d86f0657638835441871e77541165e0a2581b53b3dd657cf13ad4228d4"
|
||||
|
||||
mkdir -p /tmp/homebrew-formula
|
||||
curl -s -L $FORMULA_URL -o /tmp/homebrew-formula/cmake.rb
|
||||
|
||||
# Verify the downloaded formula
|
||||
ACTUAL_SHA256=$(shasum -a 256 /tmp/homebrew-formula/cmake.rb | cut -d ' ' -f 1)
|
||||
if [ "$ACTUAL_SHA256" != "$FORMULA_EXPECTED_SHA256" ]; then
|
||||
echo "Error: Formula checksum mismatch"
|
||||
echo "Expected: $FORMULA_EXPECTED_SHA256"
|
||||
echo "Actual: $ACTUAL_SHA256"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install cmake from the specific formula with force flag
|
||||
brew install --force /tmp/homebrew-formula/cmake.rb
|
||||
|
||||
- name: Fix git permissions on Linux
|
||||
if: ${{ runner.os == 'Linux' }}
|
||||
shell: bash
|
||||
|
||||
92
cliff.toml
92
cliff.toml
@@ -1,92 +0,0 @@
|
||||
# git-cliff ~ default configuration file
|
||||
# https://git-cliff.org/docs/configuration
|
||||
#
|
||||
# Lines starting with "#" are comments.
|
||||
# Configuration options are organized into tables and keys.
|
||||
# See documentation for more information on available options.
|
||||
|
||||
[changelog]
|
||||
# template for the changelog header
|
||||
header = """
|
||||
# Changelog\n
|
||||
All notable changes to this project will be documented in this file.\n
|
||||
"""
|
||||
# template for the changelog body
|
||||
# https://keats.github.io/tera/docs/#introduction
|
||||
body = """
|
||||
{% if version %}\
|
||||
## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }}
|
||||
{% else %}\
|
||||
## [unreleased]
|
||||
{% endif %}\
|
||||
{% for group, commits in commits | filter(attribute="merge_commit", value=false) | group_by(attribute="group") %}
|
||||
### {{ group | striptags | trim | upper_first }}
|
||||
{% for commit in commits %}
|
||||
- {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
|
||||
{% if commit.breaking %}[**breaking**] {% endif %}\
|
||||
{{ commit.message | upper_first }} {% if commit.remote.username %}by @{{ commit.remote.username }}{% endif %}\
|
||||
{% endfor %}
|
||||
{% endfor %}\n
|
||||
"""
|
||||
# template for the changelog footer
|
||||
footer = """
|
||||
<!-- generated by git-cliff -->
|
||||
"""
|
||||
# remove the leading and trailing s
|
||||
trim = true
|
||||
# postprocessors
|
||||
postprocessors = [
|
||||
# { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
|
||||
]
|
||||
# render body even when there are no releases to process
|
||||
# render_always = true
|
||||
# output file path
|
||||
output = "CHANGELOG.md"
|
||||
|
||||
[git]
|
||||
# parse the commits based on https://www.conventionalcommits.org
|
||||
conventional_commits = true
|
||||
# filter out the commits that are not conventional
|
||||
filter_unconventional = true
|
||||
# process each line of a commit as an individual commit
|
||||
split_commits = false
|
||||
# regex for preprocessing the commit messages
|
||||
commit_preprocessors = [
|
||||
# Replace issue numbers
|
||||
#{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
|
||||
# Check spelling of the commit with https://github.com/crate-ci/typos
|
||||
# If the spelling is incorrect, it will be automatically fixed.
|
||||
#{ pattern = '.*', replace_command = 'typos --write-changes -' },
|
||||
]
|
||||
# regex for parsing and grouping commits
|
||||
commit_parsers = [
|
||||
{ message = "^feat", group = "<!-- 0 -->🚀 Features" },
|
||||
{ message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
|
||||
{ message = "^doc", group = "<!-- 3 -->📚 Documentation" },
|
||||
{ message = "^perf", group = "<!-- 4 -->⚡ Performance" },
|
||||
{ message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
|
||||
{ message = "^style.*[Cc]lang-tidy auto fixes", skip = true },
|
||||
{ message = "^style", group = "<!-- 5 -->🎨 Styling" },
|
||||
{ message = "^test", group = "<!-- 6 -->🧪 Testing" },
|
||||
{ message = "^chore\\(release\\): prepare for", skip = true },
|
||||
{ message = "^chore: Commits", skip = true },
|
||||
{ message = "^chore\\(deps.*\\)", skip = true },
|
||||
{ message = "^chore\\(pr\\)", skip = true },
|
||||
{ message = "^chore\\(pull\\)", skip = true },
|
||||
{ message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
|
||||
{ body = ".*security", group = "<!-- 8 -->🛡️ Security" },
|
||||
{ message = "^revert", group = "<!-- 9 -->◀️ Revert" },
|
||||
{ message = ".*", group = "<!-- 10 -->💼 Other" },
|
||||
]
|
||||
# filter out the commits that are not matched by commit parsers
|
||||
filter_commits = false
|
||||
# sort the tags topologically
|
||||
topo_order = false
|
||||
# sort the commits inside sections by oldest/newest order
|
||||
sort_commits = "oldest"
|
||||
|
||||
ignore_tags = "^.*-[b|rc].*"
|
||||
|
||||
[remote.github]
|
||||
owner = "XRPLF"
|
||||
repo = "clio"
|
||||
@@ -23,13 +23,13 @@ class Clio(ConanFile):
|
||||
}
|
||||
|
||||
requires = [
|
||||
'boost/1.83.0',
|
||||
'boost/1.82.0',
|
||||
'cassandra-cpp-driver/2.17.0',
|
||||
'fmt/10.1.1',
|
||||
'protobuf/3.21.9',
|
||||
'grpc/1.50.1',
|
||||
'openssl/1.1.1v',
|
||||
'xrpl/2.4.0',
|
||||
'openssl/1.1.1u',
|
||||
'xrpl/2.4.0-rc4',
|
||||
'zlib/1.3.1',
|
||||
'libbacktrace/cci.20210118'
|
||||
]
|
||||
|
||||
@@ -154,7 +154,7 @@ public:
|
||||
}
|
||||
virtual ~BackendInterface() = default;
|
||||
|
||||
// TODO https://github.com/XRPLF/clio/issues/1956: Remove this hack once old ETL is removed.
|
||||
// TODO: Remove this hack once old ETL is removed.
|
||||
// Cache should not be exposed thru BackendInterface
|
||||
|
||||
/**
|
||||
@@ -648,14 +648,6 @@ public:
|
||||
virtual void
|
||||
writeAccountTransactions(std::vector<AccountTransactionsData> data) = 0;
|
||||
|
||||
/**
|
||||
* @brief Write a new account transaction.
|
||||
*
|
||||
* @param record An object representing the account transaction
|
||||
*/
|
||||
virtual void
|
||||
writeAccountTransaction(AccountTransactionsData record) = 0;
|
||||
|
||||
/**
|
||||
* @brief Write NFTs transactions.
|
||||
*
|
||||
|
||||
@@ -46,7 +46,6 @@
|
||||
#include <xrpl/protocol/LedgerHeader.h>
|
||||
#include <xrpl/protocol/nft.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
@@ -907,31 +906,19 @@ public:
|
||||
statements.reserve(data.size() * 10); // assume 10 transactions avg
|
||||
|
||||
for (auto& record : data) {
|
||||
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
|
||||
return schema_->insertAccountTx.bind(
|
||||
std::forward<decltype(account)>(account),
|
||||
std::make_tuple(record.ledgerSequence, record.transactionIndex),
|
||||
record.txHash
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
executor_.write(std::move(statements));
|
||||
}
|
||||
|
||||
void
|
||||
writeAccountTransaction(AccountTransactionsData record) override
|
||||
{
|
||||
std::vector<Statement> statements;
|
||||
statements.reserve(record.accounts.size());
|
||||
|
||||
std::ranges::transform(record.accounts, std::back_inserter(statements), [this, &record](auto&& account) {
|
||||
return schema_->insertAccountTx.bind(
|
||||
std::forward<decltype(account)>(account),
|
||||
std::make_tuple(record.ledgerSequence, record.transactionIndex),
|
||||
record.txHash
|
||||
std::transform(
|
||||
std::begin(record.accounts),
|
||||
std::end(record.accounts),
|
||||
std::back_inserter(statements),
|
||||
[this, &record](auto&& account) {
|
||||
return schema_->insertAccountTx.bind(
|
||||
std::forward<decltype(account)>(account),
|
||||
std::make_tuple(record.ledgerSequence, record.transactionIndex),
|
||||
record.txHash
|
||||
);
|
||||
}
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
executor_.write(std::move(statements));
|
||||
}
|
||||
@@ -942,7 +929,7 @@ public:
|
||||
std::vector<Statement> statements;
|
||||
statements.reserve(data.size());
|
||||
|
||||
std::ranges::transform(data, std::back_inserter(statements), [this](auto const& record) {
|
||||
std::transform(std::cbegin(data), std::cend(data), std::back_inserter(statements), [this](auto const& record) {
|
||||
return schema_->insertNFTTx.bind(
|
||||
record.tokenID, std::make_tuple(record.ledgerSequence, record.transactionIndex), record.txHash
|
||||
);
|
||||
@@ -1012,7 +999,7 @@ public:
|
||||
std::vector<Statement> statements;
|
||||
statements.reserve(data.size());
|
||||
for (auto [mptId, holder] : data)
|
||||
statements.push_back(schema_->insertMPTHolder.bind(mptId, holder));
|
||||
statements.push_back(schema_->insertMPTHolder.bind(std::move(mptId), std::move(holder)));
|
||||
|
||||
executor_.write(std::move(statements));
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ struct AccountTransactionsData {
|
||||
* @param meta The transaction metadata
|
||||
* @param txHash The transaction hash
|
||||
*/
|
||||
AccountTransactionsData(ripple::TxMeta const& meta, ripple::uint256 const& txHash)
|
||||
AccountTransactionsData(ripple::TxMeta& meta, ripple::uint256 const& txHash)
|
||||
: accounts(meta.getAffectedAccounts())
|
||||
, ledgerSequence(meta.getLgrSeq())
|
||||
, transactionIndex(meta.getIndex())
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#include "data/LedgerCache.hpp"
|
||||
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/Assert.hpp"
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
@@ -88,42 +87,6 @@ LedgerCache::update(std::vector<LedgerObject> const& objs, uint32_t seq, bool is
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
LedgerCache::update(std::vector<etlng::model::Object> const& objs, uint32_t seq)
|
||||
{
|
||||
if (disabled_)
|
||||
return;
|
||||
|
||||
std::scoped_lock const lck{mtx_};
|
||||
if (seq > latestSeq_) {
|
||||
ASSERT(
|
||||
seq == latestSeq_ + 1 || latestSeq_ == 0,
|
||||
"New sequence must be either next or first. seq = {}, latestSeq_ = {}",
|
||||
seq,
|
||||
latestSeq_
|
||||
);
|
||||
latestSeq_ = seq;
|
||||
}
|
||||
|
||||
deleted_.clear(); // previous update's deletes no longer needed
|
||||
|
||||
for (auto const& obj : objs) {
|
||||
if (!obj.data.empty()) {
|
||||
auto& e = map_[obj.key];
|
||||
if (seq > e.seq)
|
||||
e = {.seq = seq, .blob = obj.data};
|
||||
} else {
|
||||
if (map_.contains(obj.key))
|
||||
deleted_[obj.key] = map_[obj.key];
|
||||
|
||||
map_.erase(obj.key);
|
||||
if (!full_)
|
||||
deletes_.insert(obj.key);
|
||||
}
|
||||
}
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
std::optional<LedgerObject>
|
||||
LedgerCache::getSuccessor(ripple::uint256 const& key, uint32_t seq) const
|
||||
{
|
||||
@@ -176,29 +139,6 @@ LedgerCache::get(ripple::uint256 const& key, uint32_t seq) const
|
||||
return {e->second.blob};
|
||||
}
|
||||
|
||||
std::optional<Blob>
|
||||
LedgerCache::getDeleted(ripple::uint256 const& key, uint32_t seq) const
|
||||
{
|
||||
if (disabled_)
|
||||
return std::nullopt;
|
||||
|
||||
std::shared_lock const lck{mtx_};
|
||||
if (seq > latestSeq_)
|
||||
return std::nullopt;
|
||||
|
||||
++objectReqCounter_.get();
|
||||
|
||||
auto e = deleted_.find(key);
|
||||
if (e == deleted_.end())
|
||||
return std::nullopt;
|
||||
|
||||
if (seq < e->second.seq)
|
||||
return std::nullopt;
|
||||
|
||||
++objectHitCounter_.get();
|
||||
return {e->second.blob};
|
||||
}
|
||||
|
||||
void
|
||||
LedgerCache::setDisabled()
|
||||
{
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/prometheus/Bool.hpp"
|
||||
#include "util/prometheus/Counter.hpp"
|
||||
#include "util/prometheus/Label.hpp"
|
||||
@@ -30,6 +29,7 @@
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/hardened_hash.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
@@ -74,7 +74,6 @@ class LedgerCache : public LedgerCacheInterface {
|
||||
)};
|
||||
|
||||
std::map<ripple::uint256, CacheEntry> map_;
|
||||
std::map<ripple::uint256, CacheEntry> deleted_;
|
||||
|
||||
mutable std::shared_mutex mtx_;
|
||||
std::condition_variable_any cv_;
|
||||
@@ -95,17 +94,11 @@ class LedgerCache : public LedgerCacheInterface {
|
||||
|
||||
public:
|
||||
void
|
||||
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground) override;
|
||||
|
||||
void
|
||||
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) override;
|
||||
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) override;
|
||||
|
||||
std::optional<Blob>
|
||||
get(ripple::uint256 const& key, uint32_t seq) const override;
|
||||
|
||||
std::optional<Blob>
|
||||
getDeleted(ripple::uint256 const& key, uint32_t seq) const override;
|
||||
|
||||
std::optional<LedgerObject>
|
||||
getSuccessor(ripple::uint256 const& key, uint32_t seq) const override;
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
#pragma once
|
||||
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/hardened_hash.h>
|
||||
@@ -56,15 +55,6 @@ public:
|
||||
virtual void
|
||||
update(std::vector<LedgerObject> const& objs, uint32_t seq, bool isBackground = false) = 0;
|
||||
|
||||
/**
|
||||
* @brief Update the cache with new ledger objects.
|
||||
*
|
||||
* @param objs The ledger objects to update cache with
|
||||
* @param seq The sequence to update cache for
|
||||
*/
|
||||
virtual void
|
||||
update(std::vector<etlng::model::Object> const& objs, uint32_t seq) = 0;
|
||||
|
||||
/**
|
||||
* @brief Fetch a cached object by its key and sequence number.
|
||||
*
|
||||
@@ -75,16 +65,6 @@ public:
|
||||
virtual std::optional<Blob>
|
||||
get(ripple::uint256 const& key, uint32_t seq) const = 0;
|
||||
|
||||
/**
|
||||
* @brief Fetch a recently deleted object by its key and sequence number.
|
||||
*
|
||||
* @param key The key to fetch for
|
||||
* @param seq The sequence to fetch for
|
||||
* @return If found in deleted cache, will return the cached Blob; otherwise nullopt is returned
|
||||
*/
|
||||
virtual std::optional<Blob>
|
||||
getDeleted(ripple::uint256 const& key, uint32_t seq) const = 0;
|
||||
|
||||
/**
|
||||
* @brief Gets a cached successor.
|
||||
*
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include "etl/ETLService.hpp"
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "data/LedgerCache.hpp"
|
||||
#include "etl/CorruptionDetector.hpp"
|
||||
#include "etl/NetworkValidatedLedgersInterface.hpp"
|
||||
#include "feed/SubscriptionManagerInterface.hpp"
|
||||
|
||||
@@ -31,12 +31,15 @@
|
||||
|
||||
namespace etl {
|
||||
|
||||
ETLState
|
||||
tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv)
|
||||
std::optional<ETLState>
|
||||
tag_invoke(boost::json::value_to_tag<std::optional<ETLState>>, boost::json::value const& jv)
|
||||
{
|
||||
ETLState state;
|
||||
auto const& jsonObject = jv.as_object();
|
||||
|
||||
if (jsonObject.contains(JS(error)))
|
||||
return std::nullopt;
|
||||
|
||||
if (jsonObject.contains(JS(result)) && jsonObject.at(JS(result)).as_object().contains(JS(info))) {
|
||||
auto const rippledInfo = jsonObject.at(JS(result)).as_object().at(JS(info)).as_object();
|
||||
if (rippledInfo.contains(JS(network_id)))
|
||||
|
||||
@@ -20,14 +20,12 @@
|
||||
#pragma once
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "rpc/JS.hpp"
|
||||
|
||||
#include <boost/json.hpp>
|
||||
#include <boost/json/conversion.hpp>
|
||||
#include <boost/json/object.hpp>
|
||||
#include <boost/json/value.hpp>
|
||||
#include <boost/json/value_to.hpp>
|
||||
#include <xrpl/protocol/jss.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
@@ -56,9 +54,8 @@ struct ETLState {
|
||||
return std::nullopt;
|
||||
});
|
||||
|
||||
if (serverInfoRippled && not serverInfoRippled->contains(JS(error))) {
|
||||
return boost::json::value_to<ETLState>(boost::json::value(*serverInfoRippled));
|
||||
}
|
||||
if (serverInfoRippled)
|
||||
return boost::json::value_to<std::optional<ETLState>>(boost::json::value(*serverInfoRippled));
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -70,7 +67,7 @@ struct ETLState {
|
||||
* @param jv The json value to convert
|
||||
* @return The ETLState
|
||||
*/
|
||||
ETLState
|
||||
tag_invoke(boost::json::value_to_tag<ETLState>, boost::json::value const& jv);
|
||||
std::optional<ETLState>
|
||||
tag_invoke(boost::json::value_to_tag<std::optional<ETLState>>, boost::json::value const& jv);
|
||||
|
||||
} // namespace etl
|
||||
|
||||
@@ -121,7 +121,7 @@ public:
|
||||
|
||||
LOG(log_.trace()) << "Inserting transaction = " << sttx.getTransactionID();
|
||||
|
||||
ripple::TxMeta const txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
|
||||
ripple::TxMeta txMeta{sttx.getTransactionID(), ledger.seq, txn.metadata_blob()};
|
||||
|
||||
auto const [nftTxs, maybeNFT] = getNFTDataFromTx(txMeta, sttx);
|
||||
result.nfTokenTxData.insert(result.nfTokenTxData.end(), nftTxs.begin(), nftTxs.end());
|
||||
|
||||
@@ -9,10 +9,6 @@ target_sources(
|
||||
impl/Loading.cpp
|
||||
impl/Monitor.cpp
|
||||
impl/TaskManager.cpp
|
||||
impl/ext/Cache.cpp
|
||||
impl/ext/Core.cpp
|
||||
impl/ext/NFT.cpp
|
||||
impl/ext/Successor.cpp
|
||||
)
|
||||
|
||||
target_link_libraries(clio_etlng PUBLIC clio_data)
|
||||
|
||||
@@ -1,59 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/impl/ext/Cache.hpp"
|
||||
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
CacheExt::CacheExt(data::LedgerCacheInterface& cache) : cache_(cache)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
CacheExt::onLedgerData(model::LedgerData const& data) const
|
||||
{
|
||||
cache_.get().update(data.objects, data.seq);
|
||||
LOG(log_.trace()) << "got data. objects cnt = " << data.objects.size();
|
||||
}
|
||||
|
||||
void
|
||||
CacheExt::onInitialData(model::LedgerData const& data) const
|
||||
{
|
||||
LOG(log_.trace()) << "got initial data. objects cnt = " << data.objects.size();
|
||||
cache_.get().update(data.objects, data.seq);
|
||||
cache_.get().setFull();
|
||||
}
|
||||
|
||||
void
|
||||
CacheExt::onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey)
|
||||
const
|
||||
{
|
||||
LOG(log_.trace()) << "got initial objects cnt = " << objs.size();
|
||||
cache_.get().update(objs, seq);
|
||||
}
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,51 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
class CacheExt {
|
||||
std::reference_wrapper<data::LedgerCacheInterface> cache_;
|
||||
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
public:
|
||||
CacheExt(data::LedgerCacheInterface& cache);
|
||||
|
||||
void
|
||||
onLedgerData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialObjects(uint32_t seq, std::vector<model::Object> const& objs, [[maybe_unused]] std::string lastKey) const;
|
||||
};
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,83 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/impl/ext/Core.hpp"
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
CoreExt::CoreExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
CoreExt::onLedgerData(model::LedgerData const& data) const
|
||||
{
|
||||
LOG(log_.debug()) << "Loading ledger data for " << data.seq;
|
||||
backend_->writeLedger(data.header, auto{data.rawHeader});
|
||||
insertTransactions(data);
|
||||
}
|
||||
|
||||
void
|
||||
CoreExt::onInitialData(model::LedgerData const& data) const
|
||||
{
|
||||
LOG(log_.info()) << "Loading initial ledger data for " << data.seq;
|
||||
backend_->writeLedger(data.header, auto{data.rawHeader});
|
||||
insertTransactions(data);
|
||||
}
|
||||
|
||||
void
|
||||
CoreExt::onInitialObject(uint32_t seq, model::Object const& obj) const
|
||||
{
|
||||
LOG(log_.trace()) << "got initial OBJ = " << obj.key << " for seq " << seq;
|
||||
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
|
||||
}
|
||||
|
||||
void
|
||||
CoreExt::onObject(uint32_t seq, model::Object const& obj) const
|
||||
{
|
||||
LOG(log_.trace()) << "got OBJ = " << obj.key << " for seq " << seq;
|
||||
backend_->writeLedgerObject(auto{obj.keyRaw}, seq, auto{obj.dataRaw});
|
||||
}
|
||||
|
||||
void
|
||||
CoreExt::insertTransactions(model::LedgerData const& data) const
|
||||
{
|
||||
for (auto const& txn : data.transactions) {
|
||||
LOG(log_.trace()) << "Inserting transaction = " << txn.sttx.getTransactionID();
|
||||
|
||||
backend_->writeAccountTransaction({txn.meta, txn.sttx.getTransactionID()});
|
||||
backend_->writeTransaction(
|
||||
auto{txn.key},
|
||||
data.seq,
|
||||
data.header.closeTime.time_since_epoch().count(), // This is why we can't use 'onTransaction'
|
||||
auto{txn.raw},
|
||||
auto{txn.metaRaw}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,58 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
class CoreExt {
|
||||
std::shared_ptr<BackendInterface> backend_;
|
||||
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
public:
|
||||
CoreExt(std::shared_ptr<BackendInterface> backend);
|
||||
|
||||
void
|
||||
onLedgerData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialObject(uint32_t seq, model::Object const& obj) const;
|
||||
|
||||
void
|
||||
onObject(uint32_t seq, model::Object const& obj) const;
|
||||
|
||||
private:
|
||||
void
|
||||
insertTransactions(model::LedgerData const& data) const;
|
||||
};
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,77 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/impl/ext/NFT.hpp"
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "etl/NFTHelpers.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
NFTExt::NFTExt(std::shared_ptr<BackendInterface> backend) : backend_(std::move(backend))
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
NFTExt::onLedgerData(model::LedgerData const& data) const
|
||||
{
|
||||
writeNFTs(data);
|
||||
}
|
||||
|
||||
void
|
||||
NFTExt::onInitialObject(uint32_t seq, model::Object const& obj) const
|
||||
{
|
||||
LOG(log_.trace()) << "got initial object with key = " << obj.key;
|
||||
backend_->writeNFTs(etl::getNFTDataFromObj(seq, obj.keyRaw, obj.dataRaw));
|
||||
}
|
||||
|
||||
void
|
||||
NFTExt::onInitialData(model::LedgerData const& data) const
|
||||
{
|
||||
LOG(log_.trace()) << "got initial TXS cnt = " << data.transactions.size();
|
||||
writeNFTs(data);
|
||||
}
|
||||
|
||||
void
|
||||
NFTExt::writeNFTs(model::LedgerData const& data) const
|
||||
{
|
||||
std::vector<NFTsData> nfts;
|
||||
std::vector<NFTTransactionsData> nftTxs;
|
||||
|
||||
for (auto const& tx : data.transactions) {
|
||||
auto const [txs, maybeNFT] = etl::getNFTDataFromTx(tx.meta, tx.sttx);
|
||||
nftTxs.insert(nftTxs.end(), txs.begin(), txs.end());
|
||||
if (maybeNFT)
|
||||
nfts.push_back(*maybeNFT);
|
||||
}
|
||||
|
||||
// This is uniqued so that we only write latest modification (as in previous implementation)
|
||||
backend_->writeNFTs(etl::getUniqueNFTsDatas(nfts));
|
||||
backend_->writeNFTTransactions(nftTxs);
|
||||
}
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,56 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "etl/NFTHelpers.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
class NFTExt {
|
||||
std::shared_ptr<BackendInterface> backend_;
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
public:
|
||||
NFTExt(std::shared_ptr<BackendInterface> backend);
|
||||
|
||||
void
|
||||
onLedgerData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialObject(uint32_t seq, model::Object const& obj) const;
|
||||
|
||||
void
|
||||
onInitialData(model::LedgerData const& data) const;
|
||||
|
||||
private:
|
||||
void
|
||||
writeNFTs(model::LedgerData const& data) const;
|
||||
};
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,222 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/impl/ext/Successor.hpp"
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/Assert.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/strHex.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <ranges>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
SuccessorExt::SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache)
|
||||
: backend_(std::move(backend)), cache_(cache)
|
||||
{
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::onInitialData(model::LedgerData const& data) const
|
||||
{
|
||||
ASSERT(cache_.get().isFull(), "Cache must be full at this point");
|
||||
ASSERT(data.edgeKeys.has_value(), "Expecting to have edge keys on initial data load");
|
||||
ASSERT(data.objects.empty(), "Should not have objects from initial data");
|
||||
writeSuccessors(data.seq);
|
||||
writeEdgeKeys(data.seq, data.edgeKeys.value());
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::onInitialObjects(
|
||||
uint32_t seq,
|
||||
[[maybe_unused]] std::vector<model::Object> const& objs,
|
||||
std::string lastKey
|
||||
) const
|
||||
{
|
||||
for (auto const& obj : objs) {
|
||||
if (!lastKey.empty())
|
||||
backend_->writeSuccessor(std::move(lastKey), seq, auto{obj.keyRaw});
|
||||
lastKey = obj.keyRaw;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::onLedgerData(model::LedgerData const& data) const
|
||||
{
|
||||
namespace vs = std::views;
|
||||
|
||||
LOG(log_.info()) << "Received ledger data for successor ext; obj cnt = " << data.objects.size()
|
||||
<< "; got successors = " << data.successors.has_value() << "; cache is "
|
||||
<< (cache_.get().isFull() ? "FULL" : "Not full");
|
||||
|
||||
auto filteredObjects = data.objects //
|
||||
| vs::filter([](auto const& obj) { return obj.type != model::Object::ModType::Modified; });
|
||||
|
||||
if (data.successors.has_value()) {
|
||||
for (auto const& successor : data.successors.value())
|
||||
writeIncludedSuccessor(data.seq, successor);
|
||||
|
||||
for (auto const& obj : filteredObjects)
|
||||
writeIncludedSuccessor(data.seq, obj);
|
||||
} else {
|
||||
if (not cache_.get().isFull() or cache_.get().latestLedgerSequence() != data.seq)
|
||||
throw std::logic_error("Cache is not full, but object neighbors were not included");
|
||||
|
||||
for (auto const& obj : filteredObjects)
|
||||
updateSuccessorFromCache(data.seq, obj);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const
|
||||
{
|
||||
auto firstBook = succ.firstBook;
|
||||
if (firstBook.empty())
|
||||
firstBook = uint256ToString(data::kLAST_KEY);
|
||||
|
||||
backend_->writeSuccessor(auto{succ.bookBase}, seq, std::move(firstBook));
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const
|
||||
{
|
||||
ASSERT(obj.type != model::Object::ModType::Modified, "Attempt to write successor for a modified object");
|
||||
|
||||
// TODO: perhaps make these optionals inside of obj and move value_or here
|
||||
auto pred = obj.predecessor;
|
||||
auto succ = obj.successor;
|
||||
|
||||
if (obj.type == model::Object::ModType::Deleted) {
|
||||
backend_->writeSuccessor(std::move(pred), seq, std::move(succ));
|
||||
} else if (obj.type == model::Object::ModType::Created) {
|
||||
backend_->writeSuccessor(std::move(pred), seq, auto{obj.keyRaw});
|
||||
backend_->writeSuccessor(auto{obj.keyRaw}, seq, std::move(succ));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const
|
||||
{
|
||||
auto const lb =
|
||||
cache_.get().getPredecessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kFIRST_KEY, .blob = {}});
|
||||
auto const ub =
|
||||
cache_.get().getSuccessor(obj.key, seq).value_or(data::LedgerObject{.key = data::kLAST_KEY, .blob = {}});
|
||||
|
||||
auto checkBookBase = false;
|
||||
auto const isDeleted = obj.data.empty();
|
||||
|
||||
if (isDeleted) {
|
||||
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(ub.key));
|
||||
} else {
|
||||
backend_->writeSuccessor(uint256ToString(lb.key), seq, uint256ToString(obj.key));
|
||||
backend_->writeSuccessor(uint256ToString(obj.key), seq, uint256ToString(ub.key));
|
||||
}
|
||||
|
||||
if (isDeleted) {
|
||||
auto const old = cache_.get().getDeleted(obj.key, seq - 1);
|
||||
ASSERT(old.has_value(), "Deleted object {} must be in cache", ripple::strHex(obj.key));
|
||||
|
||||
checkBookBase = isBookDir(obj.key, *old);
|
||||
} else {
|
||||
checkBookBase = isBookDir(obj.key, obj.data);
|
||||
}
|
||||
|
||||
if (checkBookBase) {
|
||||
auto const current = cache_.get().get(obj.key, seq);
|
||||
auto const bookBase = getBookBase(obj.key);
|
||||
|
||||
if (isDeleted and not current.has_value()) {
|
||||
updateBookSuccessor(cache_.get().getSuccessor(bookBase, seq), seq, bookBase);
|
||||
} else if (current.has_value()) {
|
||||
auto const successor = cache_.get().getSuccessor(bookBase, seq);
|
||||
ASSERT(successor.has_value(), "Book base must have a successor for seq = {}", seq);
|
||||
|
||||
if (successor->key == obj.key) {
|
||||
updateBookSuccessor(successor, seq, bookBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::updateBookSuccessor(
|
||||
std::optional<data::LedgerObject> const& maybeSuccessor,
|
||||
auto seq,
|
||||
ripple::uint256 const& bookBase
|
||||
) const
|
||||
{
|
||||
if (maybeSuccessor.has_value()) {
|
||||
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(maybeSuccessor->key));
|
||||
} else {
|
||||
backend_->writeSuccessor(uint256ToString(bookBase), seq, uint256ToString(data::kLAST_KEY));
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::writeSuccessors(uint32_t seq) const
|
||||
{
|
||||
ripple::uint256 prev = data::kFIRST_KEY;
|
||||
while (auto cur = cache_.get().getSuccessor(prev, seq)) {
|
||||
if (prev == data::kFIRST_KEY)
|
||||
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(cur->key));
|
||||
|
||||
if (isBookDir(cur->key, cur->blob)) {
|
||||
auto base = getBookBase(cur->key);
|
||||
|
||||
// make sure the base is not an actual object
|
||||
if (not cache_.get().get(base, seq)) {
|
||||
auto succ = cache_.get().getSuccessor(base, seq);
|
||||
ASSERT(succ.has_value(), "Book base {} must have a successor", ripple::strHex(base));
|
||||
|
||||
if (succ->key == cur->key)
|
||||
backend_->writeSuccessor(uint256ToString(base), seq, uint256ToString(cur->key));
|
||||
}
|
||||
}
|
||||
|
||||
prev = cur->key;
|
||||
}
|
||||
|
||||
backend_->writeSuccessor(uint256ToString(prev), seq, uint256ToString(data::kLAST_KEY));
|
||||
}
|
||||
|
||||
void
|
||||
SuccessorExt::writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const
|
||||
{
|
||||
for (auto const& key : edgeKeys) {
|
||||
auto succ = cache_.get().getSuccessor(*ripple::uint256::fromVoidChecked(key), seq);
|
||||
if (succ)
|
||||
backend_->writeSuccessor(auto{key}, seq, uint256ToString(succ->key));
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -1,82 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "data/BackendInterface.hpp"
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "util/log/Logger.hpp"
|
||||
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/basics/strHex.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace etlng::impl {
|
||||
|
||||
class SuccessorExt {
|
||||
std::shared_ptr<BackendInterface> backend_;
|
||||
std::reference_wrapper<data::LedgerCacheInterface> cache_;
|
||||
|
||||
util::Logger log_{"ETL"};
|
||||
|
||||
public:
|
||||
SuccessorExt(std::shared_ptr<BackendInterface> backend, data::LedgerCacheInterface& cache);
|
||||
|
||||
void
|
||||
onInitialData(model::LedgerData const& data) const;
|
||||
|
||||
void
|
||||
onInitialObjects(uint32_t seq, [[maybe_unused]] std::vector<model::Object> const& objs, std::string lastKey) const;
|
||||
|
||||
void
|
||||
onLedgerData(model::LedgerData const& data) const;
|
||||
|
||||
private:
|
||||
void
|
||||
writeIncludedSuccessor(uint32_t seq, model::BookSuccessor const& succ) const;
|
||||
|
||||
void
|
||||
writeIncludedSuccessor(uint32_t seq, model::Object const& obj) const;
|
||||
|
||||
void
|
||||
updateSuccessorFromCache(uint32_t seq, model::Object const& obj) const;
|
||||
|
||||
void
|
||||
updateBookSuccessor(
|
||||
std::optional<data::LedgerObject> const& maybeSuccessor,
|
||||
auto seq,
|
||||
ripple::uint256 const& bookBase
|
||||
) const;
|
||||
|
||||
void
|
||||
writeSuccessors(uint32_t seq) const;
|
||||
|
||||
void
|
||||
writeEdgeKeys(std::uint32_t seq, auto const& edgeKeys) const;
|
||||
};
|
||||
|
||||
} // namespace etlng::impl
|
||||
@@ -385,7 +385,7 @@ insertDeliverMaxAlias(boost::json::object& txJson, std::uint32_t const apiVersio
|
||||
{
|
||||
if (txJson.contains(JS(TransactionType)) and txJson.at(JS(TransactionType)).is_string() and
|
||||
txJson.at(JS(TransactionType)).as_string() == JS(Payment) and txJson.contains(JS(Amount))) {
|
||||
txJson.insert_or_assign(JS(DeliverMax), txJson[JS(Amount)]);
|
||||
txJson[JS(DeliverMax)] = txJson[JS(Amount)];
|
||||
if (apiVersion > 1)
|
||||
txJson.erase(JS(Amount));
|
||||
}
|
||||
|
||||
@@ -102,12 +102,6 @@ AccountLinesHandler::addLine(
|
||||
line.qualityIn = lineQualityIn;
|
||||
line.qualityOut = lineQualityOut;
|
||||
|
||||
if (lineNoRipple)
|
||||
line.noRipple = true;
|
||||
|
||||
if (lineNoRipplePeer)
|
||||
line.noRipplePeer = true;
|
||||
|
||||
if (lineAuth)
|
||||
line.authorized = true;
|
||||
|
||||
@@ -126,6 +120,8 @@ AccountLinesHandler::addLine(
|
||||
if (lineDeepFreezePeer)
|
||||
line.deepFreezePeer = true;
|
||||
|
||||
line.noRipple = lineNoRipple;
|
||||
line.noRipplePeer = lineNoRipplePeer;
|
||||
lines.push_back(line);
|
||||
}
|
||||
|
||||
@@ -261,11 +257,8 @@ tag_invoke(
|
||||
{JS(quality_out), line.qualityOut},
|
||||
};
|
||||
|
||||
if (line.noRipple)
|
||||
obj[JS(no_ripple)] = *(line.noRipple);
|
||||
|
||||
if (line.noRipplePeer)
|
||||
obj[JS(no_ripple_peer)] = *(line.noRipplePeer);
|
||||
obj[JS(no_ripple)] = line.noRipple;
|
||||
obj[JS(no_ripple_peer)] = line.noRipplePeer;
|
||||
|
||||
if (line.authorized)
|
||||
obj[JS(authorized)] = *(line.authorized);
|
||||
|
||||
@@ -70,8 +70,8 @@ public:
|
||||
std::string limitPeer;
|
||||
uint32_t qualityIn{};
|
||||
uint32_t qualityOut{};
|
||||
std::optional<bool> noRipple;
|
||||
std::optional<bool> noRipplePeer;
|
||||
bool noRipple{};
|
||||
bool noRipplePeer{};
|
||||
std::optional<bool> authorized;
|
||||
std::optional<bool> peerAuthorized;
|
||||
std::optional<bool> freeze;
|
||||
|
||||
@@ -10,6 +10,7 @@ target_sources(
|
||||
ng/impl/ErrorHandling.cpp
|
||||
ng/impl/ConnectionHandler.cpp
|
||||
ng/impl/ServerSslContext.cpp
|
||||
ng/impl/WsConnection.cpp
|
||||
ng/Request.cpp
|
||||
ng/Response.cpp
|
||||
ng/Server.cpp
|
||||
|
||||
@@ -46,7 +46,6 @@
|
||||
#include <boost/system/system_error.hpp>
|
||||
#include <fmt/core.h>
|
||||
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
@@ -137,19 +136,13 @@ makeConnection(
|
||||
if (not sslContext.has_value())
|
||||
return std::unexpected{"Error creating a connection: SSL is not supported by this server"};
|
||||
|
||||
auto sslConnection = std::make_unique<impl::SslHttpConnection>(
|
||||
connection = std::make_unique<impl::SslHttpConnection>(
|
||||
std::move(sslDetectionResult.socket),
|
||||
std::move(ip),
|
||||
std::move(sslDetectionResult.buffer),
|
||||
*sslContext,
|
||||
tagDecoratorFactory
|
||||
);
|
||||
sslConnection->setTimeout(std::chrono::seconds{10});
|
||||
auto const maybeError = sslConnection->sslHandshake(yield);
|
||||
if (maybeError.has_value())
|
||||
return std::unexpected{fmt::format("SSL handshake error: {}", maybeError->message())};
|
||||
|
||||
connection = std::move(sslConnection);
|
||||
} else {
|
||||
connection = std::make_unique<impl::PlainHttpConnection>(
|
||||
std::move(sslDetectionResult.socket),
|
||||
@@ -171,6 +164,7 @@ makeConnection(
|
||||
std::expected<ConnectionPtr, std::string>
|
||||
tryUpgradeConnection(
|
||||
impl::UpgradableConnectionPtr connection,
|
||||
std::optional<boost::asio::ssl::context>& sslContext,
|
||||
util::TagDecoratorFactory& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
)
|
||||
@@ -183,7 +177,7 @@ tryUpgradeConnection(
|
||||
}
|
||||
|
||||
if (*expectedIsUpgrade) {
|
||||
auto expectedUpgradedConnection = connection->upgrade(tagDecoratorFactory, yield);
|
||||
auto expectedUpgradedConnection = connection->upgrade(sslContext, tagDecoratorFactory, yield);
|
||||
if (expectedUpgradedConnection.has_value())
|
||||
return std::move(expectedUpgradedConnection).value();
|
||||
|
||||
@@ -322,7 +316,8 @@ Server::handleConnection(boost::asio::ip::tcp::socket socket, boost::asio::yield
|
||||
return;
|
||||
}
|
||||
|
||||
auto connection = tryUpgradeConnection(std::move(connectionExpected).value(), tagDecoratorFactory_, yield);
|
||||
auto connection =
|
||||
tryUpgradeConnection(std::move(connectionExpected).value(), sslContext_, tagDecoratorFactory_, yield);
|
||||
if (not connection.has_value()) {
|
||||
LOG(log_.info()) << connection.error();
|
||||
return;
|
||||
|
||||
@@ -28,12 +28,10 @@
|
||||
#include "web/ng/impl/Concepts.hpp"
|
||||
#include "web/ng/impl/WsConnection.hpp"
|
||||
|
||||
#include <boost/asio/buffer.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/asio/ssl/context.hpp>
|
||||
#include <boost/asio/ssl/stream.hpp>
|
||||
#include <boost/asio/ssl/stream_base.hpp>
|
||||
#include <boost/beast/core/basic_stream.hpp>
|
||||
#include <boost/beast/core/error.hpp>
|
||||
#include <boost/beast/core/flat_buffer.hpp>
|
||||
@@ -59,7 +57,11 @@ public:
|
||||
isUpgradeRequested(boost::asio::yield_context yield) = 0;
|
||||
|
||||
virtual std::expected<ConnectionPtr, Error>
|
||||
upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) = 0;
|
||||
upgrade(
|
||||
std::optional<boost::asio::ssl::context>& sslContext,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
) = 0;
|
||||
|
||||
virtual std::optional<Error>
|
||||
sendRaw(
|
||||
@@ -102,22 +104,6 @@ public:
|
||||
{
|
||||
}
|
||||
|
||||
std::optional<Error>
|
||||
sslHandshake(boost::asio::yield_context yield)
|
||||
requires IsSslTcpStream<StreamType>
|
||||
{
|
||||
boost::system::error_code error;
|
||||
boost::beast::get_lowest_layer(stream_).expires_after(timeout_);
|
||||
auto const bytesUsed =
|
||||
stream_.async_handshake(boost::asio::ssl::stream_base::server, buffer_.cdata(), yield[error]);
|
||||
if (error)
|
||||
return error;
|
||||
|
||||
buffer_.consume(bytesUsed);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
bool
|
||||
wasUpgraded() const override
|
||||
{
|
||||
@@ -197,18 +183,35 @@ public:
|
||||
}
|
||||
|
||||
std::expected<ConnectionPtr, Error>
|
||||
upgrade(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield) override
|
||||
upgrade(
|
||||
[[maybe_unused]] std::optional<boost::asio::ssl::context>& sslContext,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
) override
|
||||
{
|
||||
ASSERT(request_.has_value(), "Request must be present to upgrade the connection");
|
||||
|
||||
return makeWsConnection(
|
||||
std::move(stream_),
|
||||
std::move(ip_),
|
||||
std::move(buffer_),
|
||||
std::move(request_).value(),
|
||||
tagDecoratorFactory,
|
||||
yield
|
||||
);
|
||||
if constexpr (IsSslTcpStream<StreamType>) {
|
||||
ASSERT(sslContext.has_value(), "SSL context must be present to upgrade the connection");
|
||||
return makeSslWsConnection(
|
||||
boost::beast::get_lowest_layer(stream_).release_socket(),
|
||||
std::move(ip_),
|
||||
std::move(buffer_),
|
||||
std::move(request_).value(),
|
||||
sslContext.value(),
|
||||
tagDecoratorFactory,
|
||||
yield
|
||||
);
|
||||
} else {
|
||||
return makePlainWsConnection(
|
||||
stream_.release_socket(),
|
||||
std::move(ip_),
|
||||
std::move(buffer_),
|
||||
std::move(request_).value(),
|
||||
tagDecoratorFactory,
|
||||
yield
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private:
|
||||
|
||||
77
src/web/ng/impl/WsConnection.cpp
Normal file
77
src/web/ng/impl/WsConnection.cpp
Normal file
@@ -0,0 +1,77 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2024, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "web/ng/impl/WsConnection.hpp"
|
||||
|
||||
#include "util/Taggable.hpp"
|
||||
#include "web/ng/Error.hpp"
|
||||
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/asio/ssl/context.hpp>
|
||||
#include <boost/beast/core/flat_buffer.hpp>
|
||||
#include <boost/beast/http/message.hpp>
|
||||
#include <boost/beast/http/string_body.hpp>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace web::ng::impl {
|
||||
|
||||
std::expected<std::unique_ptr<PlainWsConnection>, Error>
|
||||
makePlainWsConnection(
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::beast::http::request<boost::beast::http::string_body> request,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
)
|
||||
{
|
||||
auto connection = std::make_unique<PlainWsConnection>(
|
||||
std::move(socket), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
|
||||
);
|
||||
auto maybeError = connection->performHandshake(yield);
|
||||
if (maybeError.has_value())
|
||||
return std::unexpected{maybeError.value()};
|
||||
return connection;
|
||||
}
|
||||
|
||||
std::expected<std::unique_ptr<SslWsConnection>, Error>
|
||||
makeSslWsConnection(
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::beast::http::request<boost::beast::http::string_body> request,
|
||||
boost::asio::ssl::context& sslContext,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
)
|
||||
{
|
||||
auto connection = std::make_unique<SslWsConnection>(
|
||||
std::move(socket), std::move(ip), std::move(buffer), sslContext, std::move(request), tagDecoratorFactory
|
||||
);
|
||||
auto maybeError = connection->performHandshake(yield);
|
||||
if (maybeError.has_value())
|
||||
return std::unexpected{maybeError.value()};
|
||||
return connection;
|
||||
}
|
||||
|
||||
} // namespace web::ng::impl
|
||||
@@ -68,14 +68,31 @@ class WsConnection : public WsConnectionBase {
|
||||
|
||||
public:
|
||||
WsConnection(
|
||||
StreamType&& stream,
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::beast::http::request<boost::beast::http::string_body> initialRequest,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory
|
||||
)
|
||||
requires IsTcpStream<StreamType>
|
||||
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
|
||||
, stream_(std::move(stream))
|
||||
, stream_(std::move(socket))
|
||||
, initialRequest_(std::move(initialRequest))
|
||||
{
|
||||
setupWsStream();
|
||||
}
|
||||
|
||||
WsConnection(
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::asio::ssl::context& sslContext,
|
||||
boost::beast::http::request<boost::beast::http::string_body> initialRequest,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory
|
||||
)
|
||||
requires IsSslTcpStream<StreamType>
|
||||
: WsConnectionBase(std::move(ip), std::move(buffer), tagDecoratorFactory)
|
||||
, stream_(std::move(socket), sslContext)
|
||||
, initialRequest_(std::move(initialRequest))
|
||||
{
|
||||
setupWsStream();
|
||||
@@ -172,24 +189,25 @@ private:
|
||||
using PlainWsConnection = WsConnection<boost::beast::tcp_stream>;
|
||||
using SslWsConnection = WsConnection<boost::asio::ssl::stream<boost::beast::tcp_stream>>;
|
||||
|
||||
template <typename StreamType>
|
||||
std::expected<std::unique_ptr<WsConnection<StreamType>>, Error>
|
||||
makeWsConnection(
|
||||
StreamType&& stream,
|
||||
std::expected<std::unique_ptr<PlainWsConnection>, Error>
|
||||
makePlainWsConnection(
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::beast::http::request<boost::beast::http::string_body> request,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
)
|
||||
{
|
||||
auto connection = std::make_unique<WsConnection<StreamType>>(
|
||||
std::forward<StreamType>(stream), std::move(ip), std::move(buffer), std::move(request), tagDecoratorFactory
|
||||
);
|
||||
auto maybeError = connection->performHandshake(yield);
|
||||
if (maybeError.has_value())
|
||||
return std::unexpected{maybeError.value()};
|
||||
return connection;
|
||||
}
|
||||
);
|
||||
|
||||
std::expected<std::unique_ptr<SslWsConnection>, Error>
|
||||
makeSslWsConnection(
|
||||
boost::asio::ip::tcp::socket socket,
|
||||
std::string ip,
|
||||
boost::beast::flat_buffer buffer,
|
||||
boost::beast::http::request<boost::beast::http::string_body> request,
|
||||
boost::asio::ssl::context& sslContext,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield
|
||||
);
|
||||
|
||||
} // namespace web::ng::impl
|
||||
|
||||
@@ -19,31 +19,134 @@
|
||||
|
||||
#include "util/BinaryTestObject.hpp"
|
||||
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "etlng/impl/Extraction.hpp"
|
||||
#include "util/StringUtils.hpp"
|
||||
#include "util/TestObject.hpp"
|
||||
|
||||
#include <gtest/gtest.h>
|
||||
#include <org/xrpl/rpc/v1/ledger.pb.h>
|
||||
#include <xrpl/basics/Blob.h>
|
||||
#include <xrpl/basics/StringUtilities.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
|
||||
#include <xrpl/protocol/AccountID.h>
|
||||
#include <xrpl/protocol/STTx.h>
|
||||
#include <xrpl/protocol/Serializer.h>
|
||||
#include <xrpl/protocol/TxFormats.h>
|
||||
#include <xrpl/protocol/TxMeta.h>
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
namespace {
|
||||
|
||||
constinit auto const kSEQ = 30;
|
||||
|
||||
constinit auto const kTXN_HEX =
|
||||
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
|
||||
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
|
||||
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
|
||||
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
|
||||
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
|
||||
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73C";
|
||||
|
||||
constinit auto const kTXN_META =
|
||||
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
|
||||
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
|
||||
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
|
||||
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
|
||||
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
|
||||
"C200000028751868747470733A2F2F677265677765697362726F642E63"
|
||||
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
|
||||
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
|
||||
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
|
||||
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
|
||||
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
|
||||
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
|
||||
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
|
||||
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
|
||||
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
|
||||
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
|
||||
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
|
||||
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
|
||||
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
|
||||
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
|
||||
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
|
||||
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
|
||||
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
|
||||
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
|
||||
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
|
||||
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
|
||||
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
|
||||
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
|
||||
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
|
||||
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
|
||||
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
|
||||
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
|
||||
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
|
||||
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
|
||||
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
|
||||
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
|
||||
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
|
||||
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
|
||||
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
|
||||
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
|
||||
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
|
||||
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
|
||||
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
|
||||
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
|
||||
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
|
||||
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
|
||||
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
|
||||
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
|
||||
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
|
||||
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
|
||||
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
|
||||
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
|
||||
"00000029751868747470733A2F2F677265677765697362726F642E636F"
|
||||
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
|
||||
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
|
||||
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
|
||||
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
|
||||
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
|
||||
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
|
||||
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
|
||||
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
|
||||
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
|
||||
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
|
||||
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
|
||||
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
|
||||
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
|
||||
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
|
||||
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
|
||||
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
|
||||
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
|
||||
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
|
||||
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
|
||||
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
|
||||
"B100000015751868747470733A2F2F677265677765697362726F642E63"
|
||||
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
|
||||
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
|
||||
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
|
||||
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
|
||||
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
|
||||
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
|
||||
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
|
||||
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
|
||||
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
|
||||
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
|
||||
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
|
||||
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73CE1E1F1031000";
|
||||
|
||||
constinit auto const kRAW_HEADER =
|
||||
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
|
||||
"DD733898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A"
|
||||
@@ -56,27 +159,27 @@ constinit auto const kRAW_HEADER =
|
||||
namespace util {
|
||||
|
||||
std::pair<std::string, std::string>
|
||||
createNftTxAndMetaBlobs(std::string metaStr, std::string txnStr)
|
||||
createNftTxAndMetaBlobs()
|
||||
{
|
||||
return {hexStringToBinaryString(metaStr), hexStringToBinaryString(txnStr)};
|
||||
return {hexStringToBinaryString(kTXN_META), hexStringToBinaryString(kTXN_HEX)};
|
||||
}
|
||||
|
||||
std::pair<ripple::STTx, ripple::TxMeta>
|
||||
createNftTxAndMeta(std::string hashStr, std::string metaStr, std::string txnStr)
|
||||
createNftTxAndMeta()
|
||||
{
|
||||
ripple::uint256 hash;
|
||||
EXPECT_TRUE(hash.parseHex(hashStr));
|
||||
EXPECT_TRUE(hash.parseHex("6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E"));
|
||||
|
||||
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs(metaStr, txnStr);
|
||||
auto const [metaBlob, txnBlob] = createNftTxAndMetaBlobs();
|
||||
|
||||
ripple::SerialIter it{txnBlob.data(), txnBlob.size()};
|
||||
return {ripple::STTx{it}, ripple::TxMeta{hash, kSEQ, metaBlob}};
|
||||
}
|
||||
|
||||
etlng::model::Transaction
|
||||
createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr, std::string txnStr)
|
||||
createTransaction(ripple::TxType type)
|
||||
{
|
||||
auto const [sttx, meta] = createNftTxAndMeta(hashStr, metaStr, txnStr);
|
||||
auto const [sttx, meta] = createNftTxAndMeta();
|
||||
return {
|
||||
.raw = "",
|
||||
.metaRaw = "",
|
||||
@@ -89,9 +192,10 @@ createTransaction(ripple::TxType type, std::string hashStr, std::string metaStr,
|
||||
}
|
||||
|
||||
etlng::model::Object
|
||||
createObject(etlng::model::Object::ModType modType, std::string key)
|
||||
createObject()
|
||||
{
|
||||
// random object taken from initial ledger load
|
||||
static constinit auto const kOBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
|
||||
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
|
||||
static constinit auto const kOBJ_BLOB =
|
||||
@@ -100,63 +204,13 @@ createObject(etlng::model::Object::ModType modType, std::string key)
|
||||
"000000000000016680000000000000004C4A505900000000000000000000000000000000368480B7780E3DCF5D062A7BB54129F42F"
|
||||
"8BB63367D6C38D7EA4C680004C4A505900000000000000000000000000000000C8056BA4E36038A8A0D2C0A86963153E95A84D56";
|
||||
|
||||
return {
|
||||
.key = binaryStringToUint256(hexStringToBinaryString(key)),
|
||||
.keyRaw = hexStringToBinaryString(key),
|
||||
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
|
||||
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
|
||||
.successor = hexStringToBinaryString(kOBJ_SUCC),
|
||||
.predecessor = hexStringToBinaryString(kOBJ_PRED),
|
||||
.type = modType,
|
||||
};
|
||||
}
|
||||
|
||||
etlng::model::Object
|
||||
createObjectWithBookBase(etlng::model::Object::ModType modType, std::string key)
|
||||
{
|
||||
// random object taken from initial ledger load
|
||||
static constinit auto const kOBJ_PRED = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960A";
|
||||
static constinit auto const kOBJ_SUCC = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F";
|
||||
static constinit auto const kOBJ_BLOB =
|
||||
"11006422000000022505A681E855B4E076DD06D6D583804F9DC94F641337ECB97F71860300EEC17E530A2001D6C9583FFBFAD704E299BE"
|
||||
"3E544090ECCB12AF45FD03CAEEA852E5048E57F48FD45B505A0008138882D0F98C64A1A0E6D15053589771AD08B8C13D5384FBDAE20000"
|
||||
"0948011320AC38AE866862CF5A8AF3578C600CEE8BFB894596584B60C0FFA7D22248E33CC3";
|
||||
|
||||
return {
|
||||
.key = binaryStringToUint256(hexStringToBinaryString(key)),
|
||||
.keyRaw = hexStringToBinaryString(key),
|
||||
.data = modType == etlng::model::Object::ModType::Deleted ? ripple::Blob{} : *ripple::strUnHex(kOBJ_BLOB),
|
||||
.dataRaw = modType == etlng::model::Object::ModType::Deleted ? "" : hexStringToBinaryString(kOBJ_BLOB),
|
||||
.successor = hexStringToBinaryString(kOBJ_SUCC),
|
||||
.predecessor = hexStringToBinaryString(kOBJ_PRED),
|
||||
.type = modType,
|
||||
};
|
||||
}
|
||||
|
||||
etlng::model::Object
|
||||
createObjectWithTwoNFTs()
|
||||
{
|
||||
std::string const url1 = "abcd1";
|
||||
std::string const url2 = "abcd2";
|
||||
ripple::Blob const uri1Blob(url1.begin(), url1.end());
|
||||
ripple::Blob const uri2Blob(url2.begin(), url2.end());
|
||||
|
||||
constexpr auto kACCOUNT = "rM2AGCCCRb373FRuD8wHyUwUsh2dV4BW5Q";
|
||||
constexpr auto kNFT_ID = "0008013AE1CD8B79A8BCB52335CD40DE97401B2D60A828720000099B00000000";
|
||||
constexpr auto kNFT_ID2 = "05FB0EB4B899F056FA095537C5817163801F544BAFCEA39C995D76DB4D16F9DA";
|
||||
|
||||
auto const nftPage = createNftTokenPage({{kNFT_ID, url1}, {kNFT_ID2, url2}}, std::nullopt);
|
||||
auto const serializerNftPage = nftPage.getSerializer();
|
||||
|
||||
auto const account = getAccountIdWithString(kACCOUNT);
|
||||
return {
|
||||
.key = {},
|
||||
.keyRaw = std::string(reinterpret_cast<char const*>(account.data()), ripple::AccountID::size()),
|
||||
.keyRaw = hexStringToBinaryString(kOBJ_KEY),
|
||||
.data = {},
|
||||
.dataRaw =
|
||||
std::string(static_cast<char const*>(serializerNftPage.getDataPtr()), serializerNftPage.getDataLength()),
|
||||
.successor = "",
|
||||
.predecessor = "",
|
||||
.dataRaw = hexStringToBinaryString(kOBJ_BLOB),
|
||||
.successor = hexStringToBinaryString(kOBJ_SUCC),
|
||||
.predecessor = hexStringToBinaryString(kOBJ_PRED),
|
||||
.type = etlng::model::Object::ModType::Created,
|
||||
};
|
||||
}
|
||||
@@ -165,10 +219,8 @@ etlng::model::BookSuccessor
|
||||
createSuccessor()
|
||||
{
|
||||
return {
|
||||
.firstBook =
|
||||
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000000"}),
|
||||
.bookBase =
|
||||
uint256ToString(ripple::uint256{"A000000000000000000000000000000000000000000000000000000000000001"}),
|
||||
.firstBook = "A000000000000000000000000000000000000000000000000000000000000000",
|
||||
.bookBase = "A000000000000000000000000000000000000000000000000000000000000001",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -32,149 +32,17 @@
|
||||
|
||||
namespace util {
|
||||
|
||||
static constexpr auto kDEFAULT_TXN_HEX =
|
||||
"1200192200000008240011CC9B201B001F71D6202A0000000168400000"
|
||||
"000000000C7321ED475D1452031E8F9641AF1631519A58F7B8681E172E"
|
||||
"4838AA0E59408ADA1727DD74406960041F34F10E0CBB39444B4D4E577F"
|
||||
"C0B7E8D843D091C2917E96E7EE0E08B30C91413EC551A2B8A1D405E8BA"
|
||||
"34FE185D8B10C53B40928611F2DE3B746F0303751868747470733A2F2F"
|
||||
"677265677765697362726F642E636F6D81146203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73C";
|
||||
|
||||
static constexpr auto kDEFAULT_TXN_META =
|
||||
"201C00000001F8E511005025001F71B3556ED9C9459001E4F4A9121F4E"
|
||||
"07AB6D14898A5BBEF13D85C25D743540DB59F3CF566203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CFFFFFFFFFFFFFFFFFFFFFFFFE6FAEC5A00"
|
||||
"0800006203F49C21D5D6E022CB16DE3538F248662FC73C8962EFA00000"
|
||||
"0006751868747470733A2F2F677265677765697362726F642E636F6DE1"
|
||||
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C93E8B1"
|
||||
"C200000028751868747470733A2F2F677265677765697362726F642E63"
|
||||
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"9808B6B90000001D751868747470733A2F2F677265677765697362726F"
|
||||
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73C9C28BBAC00000012751868747470733A2F2F6772656777656973"
|
||||
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
|
||||
"F248662FC73CA048C0A300000007751868747470733A2F2F6772656777"
|
||||
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73CAACE82C500000029751868747470733A2F2F6772"
|
||||
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CAEEE87B80000001E751868747470733A2F"
|
||||
"2F677265677765697362726F642E636F6DE1EC5A000800006203F49C21"
|
||||
"D5D6E022CB16DE3538F248662FC73CB30E8CAF00000013751868747470"
|
||||
"733A2F2F677265677765697362726F642E636F6DE1EC5A000800006203"
|
||||
"F49C21D5D6E022CB16DE3538F248662FC73CB72E91A200000008751868"
|
||||
"747470733A2F2F677265677765697362726F642E636F6DE1EC5A000800"
|
||||
"006203F49C21D5D6E022CB16DE3538F248662FC73CC1B453C40000002A"
|
||||
"751868747470733A2F2F677265677765697362726F642E636F6DE1EC5A"
|
||||
"000800006203F49C21D5D6E022CB16DE3538F248662FC73CC5D458BB00"
|
||||
"00001F751868747470733A2F2F677265677765697362726F642E636F6D"
|
||||
"E1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CC9F4"
|
||||
"5DAE00000014751868747470733A2F2F677265677765697362726F642E"
|
||||
"636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC7"
|
||||
"3CCE1462A500000009751868747470733A2F2F67726567776569736272"
|
||||
"6F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248"
|
||||
"662FC73CD89A24C70000002B751868747470733A2F2F67726567776569"
|
||||
"7362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE35"
|
||||
"38F248662FC73CDCBA29BA00000020751868747470733A2F2F67726567"
|
||||
"7765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB"
|
||||
"16DE3538F248662FC73CE0DA2EB100000015751868747470733A2F2F67"
|
||||
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
|
||||
"E022CB16DE3538F248662FC73CE4FA33A40000000A751868747470733A"
|
||||
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
|
||||
"21D5D6E022CB16DE3538F248662FC73CF39FFABD000000217518687474"
|
||||
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
|
||||
"03F49C21D5D6E022CB16DE3538F248662FC73CF7BFFFB0000000167518"
|
||||
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
|
||||
"00006203F49C21D5D6E022CB16DE3538F248662FC73CFBE004A7000000"
|
||||
"0B751868747470733A2F2F677265677765697362726F642E636F6DE1F1"
|
||||
"E1E72200000000501A6203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"662FC73C8962EFA000000006FAEC5A000800006203F49C21D5D6E022CB"
|
||||
"16DE3538F248662FC73C8962EFA000000006751868747470733A2F2F67"
|
||||
"7265677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6"
|
||||
"E022CB16DE3538F248662FC73C93E8B1C200000028751868747470733A"
|
||||
"2F2F677265677765697362726F642E636F6DE1EC5A000800006203F49C"
|
||||
"21D5D6E022CB16DE3538F248662FC73C9808B6B90000001D7518687474"
|
||||
"70733A2F2F677265677765697362726F642E636F6DE1EC5A0008000062"
|
||||
"03F49C21D5D6E022CB16DE3538F248662FC73C9C28BBAC000000127518"
|
||||
"68747470733A2F2F677265677765697362726F642E636F6DE1EC5A0008"
|
||||
"00006203F49C21D5D6E022CB16DE3538F248662FC73CA048C0A3000000"
|
||||
"07751868747470733A2F2F677265677765697362726F642E636F6DE1EC"
|
||||
"5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAACE82C5"
|
||||
"00000029751868747470733A2F2F677265677765697362726F642E636F"
|
||||
"6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CAE"
|
||||
"EE87B80000001E751868747470733A2F2F677265677765697362726F64"
|
||||
"2E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662F"
|
||||
"C73CB30E8CAF00000013751868747470733A2F2F677265677765697362"
|
||||
"726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F2"
|
||||
"48662FC73CB72E91A200000008751868747470733A2F2F677265677765"
|
||||
"697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE"
|
||||
"3538F248662FC73CC1B453C40000002A751868747470733A2F2F677265"
|
||||
"677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022"
|
||||
"CB16DE3538F248662FC73CC5D458BB0000001F751868747470733A2F2F"
|
||||
"677265677765697362726F642E636F6DE1EC5A000800006203F49C21D5"
|
||||
"D6E022CB16DE3538F248662FC73CC9F45DAE0000001475186874747073"
|
||||
"3A2F2F677265677765697362726F642E636F6DE1EC5A000800006203F4"
|
||||
"9C21D5D6E022CB16DE3538F248662FC73CCE1462A50000000975186874"
|
||||
"7470733A2F2F677265677765697362726F642E636F6DE1EC5A00080000"
|
||||
"6203F49C21D5D6E022CB16DE3538F248662FC73CD89A24C70000002B75"
|
||||
"1868747470733A2F2F677265677765697362726F642E636F6DE1EC5A00"
|
||||
"0800006203F49C21D5D6E022CB16DE3538F248662FC73CDCBA29BA0000"
|
||||
"0020751868747470733A2F2F677265677765697362726F642E636F6DE1"
|
||||
"EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73CE0DA2E"
|
||||
"B100000015751868747470733A2F2F677265677765697362726F642E63"
|
||||
"6F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F248662FC73C"
|
||||
"E4FA33A40000000A751868747470733A2F2F677265677765697362726F"
|
||||
"642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73CEF7FF5C60000002C751868747470733A2F2F6772656777656973"
|
||||
"62726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16DE3538"
|
||||
"F248662FC73CF39FFABD00000021751868747470733A2F2F6772656777"
|
||||
"65697362726F642E636F6DE1EC5A000800006203F49C21D5D6E022CB16"
|
||||
"DE3538F248662FC73CF7BFFFB000000016751868747470733A2F2F6772"
|
||||
"65677765697362726F642E636F6DE1EC5A000800006203F49C21D5D6E0"
|
||||
"22CB16DE3538F248662FC73CFBE004A70000000B751868747470733A2F"
|
||||
"2F677265677765697362726F642E636F6DE1F1E1E1E511006125001F71"
|
||||
"B3556ED9C9459001E4F4A9121F4E07AB6D14898A5BBEF13D85C25D7435"
|
||||
"40DB59F3CF56BE121B82D5812149D633F605EB07265A80B762A365CE94"
|
||||
"883089FEEE4B955701E6240011CC9B202B0000002C6240000002540BE3"
|
||||
"ECE1E72200000000240011CC9C2D0000000A202B0000002D202C000000"
|
||||
"066240000002540BE3E081146203F49C21D5D6E022CB16DE3538F24866"
|
||||
"2FC73CE1E1F1031000";
|
||||
|
||||
static constexpr auto kDEFAULT_HASH = "6C7F69A6D25A13AC4A2E9145999F45D4674F939900017A96885FDC2757E9284E";
|
||||
static constexpr auto kDEFAULT_OBJ_KEY = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
|
||||
[[maybe_unused, nodiscard]] std::pair<std::string, std::string>
|
||||
createNftTxAndMetaBlobs(std::string metaStr = kDEFAULT_TXN_META, std::string txnStr = kDEFAULT_TXN_HEX);
|
||||
createNftTxAndMetaBlobs();
|
||||
|
||||
[[maybe_unused, nodiscard]] std::pair<ripple::STTx, ripple::TxMeta>
|
||||
createNftTxAndMeta(
|
||||
std::string hashStr = kDEFAULT_HASH,
|
||||
std::string metaStr = kDEFAULT_TXN_META,
|
||||
std::string txnStr = kDEFAULT_TXN_HEX
|
||||
);
|
||||
createNftTxAndMeta();
|
||||
|
||||
[[maybe_unused, nodiscard]] etlng::model::Transaction
|
||||
createTransaction(
|
||||
ripple::TxType type,
|
||||
std::string hashStr = kDEFAULT_HASH,
|
||||
std::string metaStr = kDEFAULT_TXN_META,
|
||||
std::string txnStr = kDEFAULT_TXN_HEX
|
||||
);
|
||||
createTransaction(ripple::TxType type);
|
||||
|
||||
[[maybe_unused, nodiscard]] etlng::model::Object
|
||||
createObject(
|
||||
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
|
||||
std::string key = kDEFAULT_OBJ_KEY
|
||||
);
|
||||
|
||||
[[maybe_unused, nodiscard]] etlng::model::Object
|
||||
createObjectWithBookBase(
|
||||
etlng::model::Object::ModType modType = etlng::model::Object::ModType::Created,
|
||||
std::string key = kDEFAULT_OBJ_KEY
|
||||
);
|
||||
|
||||
[[maybe_unused, nodiscard]] etlng::model::Object
|
||||
createObjectWithTwoNFTs();
|
||||
createObject();
|
||||
|
||||
[[maybe_unused, nodiscard]] etlng::model::BookSuccessor
|
||||
createSuccessor();
|
||||
|
||||
@@ -203,8 +203,6 @@ struct MockBackend : public BackendInterface {
|
||||
|
||||
MOCK_METHOD(void, writeAccountTransactions, (std::vector<AccountTransactionsData>), (override));
|
||||
|
||||
MOCK_METHOD(void, writeAccountTransaction, (AccountTransactionsData), (override));
|
||||
|
||||
MOCK_METHOD(void, writeNFTTransactions, (std::vector<NFTTransactionsData> const&), (override));
|
||||
|
||||
MOCK_METHOD(void, writeSuccessor, (std::string && key, std::uint32_t const, std::string&&), (override));
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
|
||||
#include "data/LedgerCacheInterface.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
@@ -42,10 +41,6 @@ struct MockLedgerCache : data::LedgerCacheInterface {
|
||||
|
||||
MOCK_METHOD(std::optional<data::Blob>, get, (ripple::uint256 const& a, uint32_t b), (const, override));
|
||||
|
||||
MOCK_METHOD(void, update, (std::vector<etlng::model::Object> const&, uint32_t), (override));
|
||||
|
||||
MOCK_METHOD(std::optional<data::Blob>, getDeleted, (ripple::uint256 const&, uint32_t), (const, override));
|
||||
|
||||
MOCK_METHOD(
|
||||
std::optional<data::LedgerObject>,
|
||||
getSuccessor,
|
||||
|
||||
@@ -67,7 +67,9 @@ struct MockHttpConnectionImpl : web::ng::impl::UpgradableConnection {
|
||||
MOCK_METHOD(
|
||||
UpgradeReturnType,
|
||||
upgrade,
|
||||
(util::TagDecoratorFactory const& tagDecoratorFactory, boost::asio::yield_context yield),
|
||||
(OptionalSslContext & sslContext,
|
||||
util::TagDecoratorFactory const& tagDecoratorFactory,
|
||||
boost::asio::yield_context yield),
|
||||
(override)
|
||||
);
|
||||
};
|
||||
|
||||
@@ -402,7 +402,7 @@ TEST_F(BackendCassandraTest, Basic)
|
||||
|
||||
ripple::uint256 hash256;
|
||||
EXPECT_TRUE(hash256.parseHex(hashHex));
|
||||
ripple::TxMeta const txMeta{hash256, lgrInfoNext.seq, metaBlob};
|
||||
ripple::TxMeta txMeta{hash256, lgrInfoNext.seq, metaBlob};
|
||||
auto accountsSet = txMeta.getAffectedAccounts();
|
||||
for (auto& a : accountsSet) {
|
||||
affectedAccounts.push_back(a);
|
||||
|
||||
@@ -45,10 +45,6 @@ target_sources(
|
||||
etlng/LoadingTests.cpp
|
||||
etlng/NetworkValidatedLedgersTests.cpp
|
||||
etlng/MonitorTests.cpp
|
||||
etlng/ext/CoreTests.cpp
|
||||
etlng/ext/CacheTests.cpp
|
||||
etlng/ext/NFTTests.cpp
|
||||
etlng/ext/SuccessorTests.cpp
|
||||
# Feed
|
||||
util/BytesConverterTests.cpp
|
||||
feed/BookChangesFeedTests.cpp
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/Models.hpp"
|
||||
#include "etlng/impl/ext/Cache.hpp"
|
||||
#include "util/BinaryTestObject.hpp"
|
||||
#include "util/MockLedgerCache.hpp"
|
||||
#include "util/MockPrometheus.hpp"
|
||||
#include "util/TestObject.hpp"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using namespace etlng::impl;
|
||||
using namespace data;
|
||||
|
||||
namespace {
|
||||
constinit auto const kSEQ = 123u;
|
||||
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
|
||||
constinit auto const kUNUSED_LAST_KEY = "unused";
|
||||
|
||||
auto
|
||||
createTestData()
|
||||
{
|
||||
auto objects = std::vector{util::createObject(), util::createObject(), util::createObject()};
|
||||
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
|
||||
return etlng::model::LedgerData{
|
||||
.transactions = {},
|
||||
.objects = std::move(objects),
|
||||
.successors = {},
|
||||
.edgeKeys = {},
|
||||
.header = header,
|
||||
.rawHeader = {},
|
||||
.seq = kSEQ
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct CacheExtTests : util::prometheus::WithPrometheus {
|
||||
protected:
|
||||
MockLedgerCache cache_;
|
||||
etlng::impl::CacheExt ext_{cache_};
|
||||
};
|
||||
|
||||
TEST_F(CacheExtTests, OnLedgerDataUpdatesCache)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(cache_, update(data.objects, data.seq));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(CacheExtTests, OnInitialDataUpdatesCacheAndSetsFull)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(cache_, update(data.objects, data.seq));
|
||||
EXPECT_CALL(cache_, setFull);
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(CacheExtTests, OnInitialObjectsUpdateCache)
|
||||
{
|
||||
auto const objects = std::vector{util::createObject(), util::createObject()};
|
||||
|
||||
EXPECT_CALL(cache_, update(objects, kSEQ));
|
||||
|
||||
ext_.onInitialObjects(kSEQ, objects, kUNUSED_LAST_KEY);
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/Models.hpp"
|
||||
#include "etlng/impl/ext/Core.hpp"
|
||||
#include "util/BinaryTestObject.hpp"
|
||||
#include "util/MockBackendTestFixture.hpp"
|
||||
#include "util/MockPrometheus.hpp"
|
||||
#include "util/TestObject.hpp"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <xrpl/protocol/TxFormats.h>
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using namespace etlng::impl;
|
||||
using namespace data;
|
||||
|
||||
namespace {
|
||||
constinit auto const kSEQ = 123u;
|
||||
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
|
||||
|
||||
auto
|
||||
createTestData()
|
||||
{
|
||||
auto transactions = std::vector{
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
|
||||
};
|
||||
|
||||
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
|
||||
return etlng::model::LedgerData{
|
||||
.transactions = std::move(transactions),
|
||||
.objects = {},
|
||||
.successors = {},
|
||||
.edgeKeys = {},
|
||||
.header = header,
|
||||
.rawHeader = {},
|
||||
.seq = kSEQ
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct CoreExtTests : util::prometheus::WithPrometheus, MockBackendTest {
|
||||
protected:
|
||||
etlng::impl::CoreExt ext_{backend_};
|
||||
};
|
||||
|
||||
TEST_F(CoreExtTests, OnLedgerDataWritesLedgerAndTransactions)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
|
||||
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
|
||||
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(CoreExtTests, OnInitialDataWritesLedgerAndTransactions)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(*backend_, writeLedger(testing::_, auto{data.rawHeader}));
|
||||
EXPECT_CALL(*backend_, writeAccountTransaction).Times(data.transactions.size());
|
||||
EXPECT_CALL(*backend_, writeTransaction).Times(data.transactions.size());
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(CoreExtTests, OnInitialObjectWritesLedgerObject)
|
||||
{
|
||||
auto const data = util::createObject();
|
||||
|
||||
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
|
||||
|
||||
ext_.onInitialObject(kSEQ, data);
|
||||
}
|
||||
|
||||
TEST_F(CoreExtTests, OnObjectWritesLedgerObject)
|
||||
{
|
||||
auto const data = util::createObject();
|
||||
|
||||
EXPECT_CALL(*backend_, writeLedgerObject(auto{data.keyRaw}, kSEQ, auto{data.dataRaw}));
|
||||
|
||||
ext_.onObject(kSEQ, data);
|
||||
}
|
||||
@@ -1,287 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "etlng/Models.hpp"
|
||||
#include "etlng/impl/ext/NFT.hpp"
|
||||
#include "util/BinaryTestObject.hpp"
|
||||
#include "util/MockBackendTestFixture.hpp"
|
||||
#include "util/MockPrometheus.hpp"
|
||||
#include "util/TestObject.hpp"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <xrpl/protocol/TxFormats.h>
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using namespace etlng::impl;
|
||||
using namespace data;
|
||||
|
||||
namespace {
|
||||
constinit auto const kSEQ = 123u;
|
||||
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
|
||||
|
||||
constinit auto const kTXN_HEX2 =
|
||||
"12001D230606B58324048A8B6F501C50E8EBCD412E6CF9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487E501DA2A1BC9A62AAEB2A"
|
||||
"2A70F895587A3FB752514AA03F8C6E7C84864653B8673E0368400000000000001E60134000000000001F09732103B8C234E0598BC26D8A3E1B"
|
||||
"FF53EB252EC0F15EA6800E4D85AA5D7CD15D76B01E744730450221009C0AFF5F3298E10ABE42894717DA46B59529A366527AA5DFC1577ADEA9"
|
||||
"B20FA70220494D1D9BFEF2AB09F4D6403AAC5B9DCC5B859DCEC1380C6D817A6EDFE7E68FD581141565EED165BA79999425204A8491C73B1301"
|
||||
"E34FF9EA7D0F7872702E63616665202D2073616C65E1F1";
|
||||
|
||||
constinit auto const kTXN_META2 =
|
||||
"201C00000040F8E51100502505A59E11552ABC2FD74D879BE58489A588838AA2BA59E1E05A48A574226CD8B6CE77998971560B639A808E3B97"
|
||||
"42A25334E5CF68EEDEDE52F54E50F4E63921C9F3C40588E426E6FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C"
|
||||
"052AB2897542697066733A2F2F6261666B726569683673796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E71673472"
|
||||
"6E747037653769757469766C796275E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D526950"
|
||||
"5679335464654A6170697974576B686141576B4D716D39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60F"
|
||||
"DDC47DABE4D52E4AD1F0A50A85CBBC00000022752E516D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169"
|
||||
"316B4C316E5255797131E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C457664644634"
|
||||
"5442427374455138484C66694867456B6A4465514D56676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4"
|
||||
"D52E4AD1F0A5608E298B000000EF752E516D4E6E56566F4250426755783848334641486F436162593647347577445758767738757455636976"
|
||||
"73764D7142E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646"
|
||||
"777A526D517472694841663277437A3567786161355256446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0"
|
||||
"A5B09C35A40000020A752E516D5168584346325738535571657633333657703572357935326141464D59564A667570534A536B6B425047786E"
|
||||
"E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B6968756000001BC752E516D5359647963353276657A7478365338673231"
|
||||
"70544651724E6E624762676A5A4B5047753836464231394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C"
|
||||
"000000D2752E516D506645624A38446D624E69794C55637441796A3473625453515937314177597A32776A595238703838416432E1EC5A0008"
|
||||
"27107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37"
|
||||
"773338734147397469725867326D626D4262596771E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F75"
|
||||
"2E516D505A733255393159764538374277635677587A63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E6"
|
||||
"4C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E21584000001EA752E516D656366646769555A6B37355574667152763474696644557257345439"
|
||||
"4571616F733231757036706E75345663E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573"
|
||||
"614345725175396F48596A71765172564B31473155504657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE"
|
||||
"11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A6854653776"
|
||||
"43624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A2F2F6261"
|
||||
"667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E6965"
|
||||
"2F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F62616679"
|
||||
"62656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E69652F36"
|
||||
"342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D616D397436"
|
||||
"5A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0FE30354A4"
|
||||
"3281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D79756E697736"
|
||||
"647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A43281629A"
|
||||
"0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D6575326934"
|
||||
"65726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A00082710"
|
||||
"AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E61776833"
|
||||
"6F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B7320"
|
||||
"2331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F516D63436D"
|
||||
"526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E72200000000501A0B639A808E"
|
||||
"3B9742A25334E5CF68EEDEDE52F54E4A69498F5B79952C052AB289501B0B639A808E3B9742A25334E5CF68EEDEDE52F54E50F4E639664EC7E5"
|
||||
"0588E658FAEC5A000827104B18F97F9209869C9E9CC33EC2AAE2864A69498F5B79952C052AB2897542697066733A2F2F6261666B7265696836"
|
||||
"73796D616974676D67616E6B67766B6B3568767A786C636463326D6876346E716734726E747037653769757469766C796275E1EC5A00082710"
|
||||
"7B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5006B75170000007B752E516D5269505679335464654A6170697974576B686141576B4D716D"
|
||||
"39335350696433587A524C564A437872537772E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A50A85CBBC00000022752E51"
|
||||
"6D596543706A427A7A5257516733527935455661725A4250316B79556A47625A7A4169316B4C316E5255797131E1EC5A000827107B87E64C88"
|
||||
"4BBFB60FDDC47DABE4D52E4AD1F0A5423F2594000001FA752E516D554C4576646446345442427374455138484C66694867456B6A4465514D56"
|
||||
"676F39486A463476656752533277E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5608E298B000000EF752E516D4E6E5656"
|
||||
"6F4250426755783848334641486F43616259364734757744575876773875745563697673764D7142E1EC5A000827107B87E64C884BBFB60FDD"
|
||||
"C47DABE4D52E4AD1F0A57A9089B80000021E752E516D5275674657434B626443325646777A526D517472694841663277437A35677861613552"
|
||||
"56446A58447A323664E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5B09C35A40000020A752E516D516858434632573853"
|
||||
"5571657633333657703572357935326141464D59564A667570534A536B6B425047786EE1EC5A000827107B87E64C884BBFB60FDDC47DABE4D5"
|
||||
"2E4AD1F0A5B6968756000001BC752E516D5359647963353276657A747836533867323170544651724E6E624762676A5A4B5047753836464231"
|
||||
"394D6768E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5C8857C6C000000D2752E516D506645624A38446D624E69794C55"
|
||||
"637441796A3473625453515937314177597A32776A595238703838416432E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5"
|
||||
"CBA20B9A00000200752E516D637342473445784A486D6B3377576873666155774E6A37773338734147397469725867326D626D4262596771E1"
|
||||
"EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D376AA0B0000016F752E516D505A733255393159764538374277635677587A"
|
||||
"63515374564C543879424D3652424E3656544751374A4C3251E1EC5A000827107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5D3E2158400"
|
||||
"0001EA752E516D656366646769555A6B373555746671527634746966445572573454394571616F733231757036706E75345663E1EC5A000827"
|
||||
"107B87E64C884BBFB60FDDC47DABE4D52E4AD1F0A5E63C76170000017B752E516D5573614345725175396F48596A71765172564B3147315550"
|
||||
"4657713558656A5A7765795170523133444D446DE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051752E"
|
||||
"516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44668A"
|
||||
"2B96DFDE11BF0817CC6DF60C4E3508D4F9BEA82100000985752E516D5847564C4C6857484E4677587455475A696F51446D6B7851624B6B6A68"
|
||||
"5465377643624D6A6533356D526562E1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64C6BA254B0585F3A8754B697066733A"
|
||||
"2F2F6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A3670717136356635"
|
||||
"6F6E69652F3236342E6A736F6EE1EC5A000827100F280C8A448F4E0D283C0F6D52EE01454FDBFD64DD9FF64C0585F3A9754A697066733A2F2F"
|
||||
"6261667962656968636836686B6834336A766A72367775727974756D3663347A6F686C74706B746261756D3763717A36707171363566356F6E"
|
||||
"69652F36342E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E63902A418D00588E43B7535697066733A2F2F516D61"
|
||||
"6D3974365A6962324E485270326869796A5A567A3964564D526B7A4A547A754A343369507672686A786344E1EC5A00082710AF1D39F2E0BB0F"
|
||||
"E30354A43281629A0B50F4E6390E12DFEA0588EA2C7549697066733A2F2F626166796265696678686C6B706F36673778376736686F336D7975"
|
||||
"6E697736647137366A67643262686136707536656C70686977743567613270712F392E6A736F6EE1EC5A00082710AF1D39F2E0BB0FE30354A4"
|
||||
"3281629A0B50F4E639153D46070588E676755C697066733A2F2F6261667962656964777370686336776E617768336F6D34356970746E776D65"
|
||||
"7532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B732050756E6B73202332362E6A736F6EE1EC5A"
|
||||
"00082710AF1D39F2E0BB0FE30354A43281629A0B50F4E6391D7D4FED0588E660755C697066733A2F2F6261667962656964777370686336776E"
|
||||
"617768336F6D34356970746E776D657532693465726D71616B723678717A37616C736E6564636F757278792F54686520427269636B73205075"
|
||||
"6E6B73202331312E6A736F6EE1EC5A00081388AF1D39F2E0BB0FE30354A43281629A0B50F4E6391DA9EEC90588E4317535697066733A2F2F51"
|
||||
"6D63436D526E65675A65586778446A6B3654523935454858596848684C3642564A4E516D6D78315A66747A3539E1F1E1E1E51100502505A4C9"
|
||||
"C9557DF04CC68DE7C5EF20DBF705221EEDB05FE3806BC3F6A35240652C3E9C97BA5A56246B3E06AB367AB9614566B6F90C718B52A4440852A4"
|
||||
"440804D409E004C90E52E6FAEC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478B72AD46100000008755E68747470733A2F2F69"
|
||||
"7066732E696F2F697066732F62616679626569656F6C7667696F71766F737436346367646873797876726962336D6265797278773477626F61"
|
||||
"617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10"
|
||||
"276700000009755E68747470733A2F2F697066732E696F2F697066732F62616679626569686D6374376A766B7236366E67337975676D33706D"
|
||||
"70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA70"
|
||||
"69E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E68747470733A2F2F697066732E696F2F697066732F6261667962656968626D6D"
|
||||
"6E626C687736656D776E6A733766787778376736376B6E6A626B6966636674787A6B66777632767162357A6C727575612F6D65746164617461"
|
||||
"2E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478FBE441630000000B755E68747470733A2F2F697066732E696F"
|
||||
"2F697066732F62616679626569686E6C77656C7965357270706963646761617678756869376D61636E697879627077667133796734626C3366"
|
||||
"6A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051"
|
||||
"752E516D654D784C43764A345248413465506B776F4635754461525A5A78535A65483758433931575A76535776644578E1EC5A00080FA03A44"
|
||||
"668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B535470"
|
||||
"31514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D51"
|
||||
"674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE58"
|
||||
"EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A7661"
|
||||
"6D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC753569"
|
||||
"7066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A0008"
|
||||
"2710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F62"
|
||||
"6D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC24F"
|
||||
"1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C783579693375"
|
||||
"706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408"
|
||||
"005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D7547724839"
|
||||
"4C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D555744524C"
|
||||
"425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB9614566"
|
||||
"B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E346958597233487341715771"
|
||||
"673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD7535697066733A"
|
||||
"2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A00081388246B"
|
||||
"3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A51567957"
|
||||
"4B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204C9"
|
||||
"12F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E47"
|
||||
"E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E357163334E"
|
||||
"753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52"
|
||||
"A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E654474"
|
||||
"6656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D5779"
|
||||
"6D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367AB9"
|
||||
"614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A743147454564"
|
||||
"4C787161513466714E637375363854677153E1F1E1E72200000000501A246B3E06AB367AB9614566B6F90C718B52A444084B237478B72AD461"
|
||||
"00000008501B246B3E06AB367AB9614566B6F90C718B52A4440852A444080D2641FC04C91315FAEC5A00081388DAA8A3AA7069E65EC1E3E557"
|
||||
"1D9B6F274B237478B72AD46100000008755E68747470733A2F2F697066732E696F2F697066732F62616679626569656F6C7667696F71766F73"
|
||||
"7436346367646873797876726962336D6265797278773477626F61617A3270656D696D63327864326D2F6D657461646174612E6A736F6EE1EC"
|
||||
"5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478CE10276700000009755E68747470733A2F2F697066732E696F2F697066732F62"
|
||||
"616679626569686D6374376A766B7236366E67337975676D33706D70336E6B68676F786474746278656F6665786C617566616C6B6E32746D69"
|
||||
"792F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B237478E4FE76610000000A755E6874747073"
|
||||
"3A2F2F697066732E696F2F697066732F6261667962656968626D6D6E626C687736656D776E6A733766787778376736376B6E6A626B69666366"
|
||||
"74787A6B66777632767162357A6C727575612F6D657461646174612E6A736F6EE1EC5A00081388DAA8A3AA7069E65EC1E3E5571D9B6F274B23"
|
||||
"7478FBE441630000000B755E68747470733A2F2F697066732E696F2F697066732F62616679626569686E6C77656C7965357270706963646761"
|
||||
"617678756869376D61636E697879627077667133796734626C33666A757235683735692F6D657461646174612E6A736F6EE1EC5A00080FA03A"
|
||||
"44668A2B96DFDE11BF0817CC6DF60C4E3508D46A3D14B70000001B752E516D58446D6452435266326A6A75437758366F7347716B6F7A4B5354"
|
||||
"7031514E38516562776F6F63376775396553E1EC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D493E8B1C200000028752E516D"
|
||||
"51674C5878767132484D6E6E4D446D6D6557486935477833565331706843544C54473132334C784857697535E1EC5A00081A043ACC61B05EAE"
|
||||
"58EC755700FFBD17A9EA4E5581530C027A22054839877535697066733A2F2F516D53454258334D48436D67706769554C387235784269526A76"
|
||||
"616D3547564C4364386E724A6B4E386164626978E1EC5A00081A043ACC61B05EAE58EC755700FFBD17A9EA4E5581536C51CD67054837CC7535"
|
||||
"697066733A2F2F516D5939344C7365465247757577447764446870436778597572657A424D6D3431376D724C35454A6758374D7848E1EC5A00"
|
||||
"082710CBDCBA9A66CC3AC24F1B77CE45DCAB1C502A6AC29808B6B80000001D7542697066733A2F2F6261666B726569637235337936706E326F"
|
||||
"626D6474706434776F6F337A35766B65737477376A64726372786B736D666C7134743335653575716B69E1EC5A00082710CBDCBA9A66CC3AC2"
|
||||
"4F1B77CE45DCAB1C502A6AC2A048C0A2000000077542697066733A2F2F6261666B72656963326C6B6933736A62657171616A366C7835796933"
|
||||
"75706770736D636773356A6733777867373666756F6A75666B366D62666465E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A444"
|
||||
"08005AC71A04C912BB7535697066733A2F2F516D5064595531374B575A676F355076516246563642504D7832765365507942767A6D75477248"
|
||||
"394C4869777973E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440800C6329D04C913367535697066733A2F2F516D55574452"
|
||||
"4C425768416661436F4B6B786B507161666652776F54584A48483953675062666A334D69626A4E46E1EC5A00081388246B3E06AB367AB96145"
|
||||
"66B6F90C718B52A44408015E3D5104C911827535697066733A2F2F516D614439647A734A385972544239576E516E3469585972334873417157"
|
||||
"71673273586543447670506B77375A6FE1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A4440801C9A8D404C911FD753569706673"
|
||||
"3A2F2F516D565947596754376A64544C54654B4B7347623737534138396943576547456B79566176474761433674394555E1EC5A0008138824"
|
||||
"6B3E06AB367AB9614566B6F90C718B52A444080235145F04C912787535697066733A2F2F516D56676A4C6D7178736F594E6F657A437A515679"
|
||||
"574B57565737417775706D4A5A6E5773547475705A4D434775E1EC5A00082710246B3E06AB367AB9614566B6F90C718B52A4440802A07FC204"
|
||||
"C912F37535697066733A2F2F516D63503174364A4832567179677131485855414A4A3558543152345474486677704D35393243384C4379426E"
|
||||
"47E1EC5A0008C350246B3E06AB367AB9614566B6F90C718B52A44408030BEB4504C9136E7535697066733A2F2F516D63587852656E35716333"
|
||||
"4E753133726D31647355707378503278576474416265386F75524A4C4A42386E4C58E1EC5A00081388246B3E06AB367AB9614566B6F90C718B"
|
||||
"52A4440803A3F51904C911BA7535697066733A2F2F516D4E717169357477776A3169356A64697562767A4A373355534A513856626B344E6544"
|
||||
"746656703171506D7861E1EC5A00081388246B3E06AB367AB9614566B6F90C718B52A44408040F609C04C912357535697066733A2F2F516D57"
|
||||
"796D716A34374A464E4765713867684C53715031625639596E3865464639547167655933534E3169694876E1EC5A00081388246B3E06AB367A"
|
||||
"B9614566B6F90C718B52A44408047ACC0704C912B07535697066733A2F2F516D565451584A38636F6B63317653614A776B65456A7431474545"
|
||||
"644C787161513466714E637375363854677153E1F1E1E1E4110064562AED34CB796DF0E82AAC7EB958158EEBF99F51AA8C96B85654DDE05206"
|
||||
"C18BBCE7220000000225059FB7B755172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235BEBBE582AED34CB796DF0E82A"
|
||||
"AC7EB958158EEBF99F51AA8C96B85654DDE05206C18BBC5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED00000051E1"
|
||||
"E1E51100612505A58BFB555F1D177245DF64F1BAC04B3EFF72C9B448710F95E86355339305264B0C6450345643CECFECC44660B31BEB70A4AE"
|
||||
"78ED0BB4B31A754CE746848150EFFC03418847E62D0000010E624000000004FF530BE1E722000000002404C9421F2D0000010D202B00000712"
|
||||
"202C00000493203204C90C7062400000000506A60B8114246B3E06AB367AB9614566B6F90C718B52A44408E1E1E41100375650E8EBCD412E6C"
|
||||
"F9D0C2EB6D38BDE1E1C83406AFCB45437DF39A8B0677A9487EE722000000002505A5B1C23400000000000000463C0000000000000000558F56"
|
||||
"F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB7"
|
||||
"2AED0000005161400000000007C02982140B639A808E3B9742A25334E5CF68EEDEDE52F54E83141565EED165BA79999425204A8491C73B1301"
|
||||
"E34FE1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC4565183948C67127DAA598D"
|
||||
"1197F3DEDC4552F64034F3B8213060B8903FD6C8A561E62D0000039C62400000000D7F518BE1E7220000000024057A8B032D0000039B202B00"
|
||||
"000290202C000000082032057A6CBC62400000000D77916281140B639A808E3B9742A25334E5CF68EEDEDE52F54E8914CEAECC5B87EA043BD9"
|
||||
"8E1B4FE8663AC59D5C3518E1E1E51100612505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1FD7B40F710AC45658"
|
||||
"7E28972F3B63D2260C0671E59593EE6C4D27AB857D1AF230F5CAA959BB3EACE624048A8B6F624000001B84AABF6AE1E7220000000024048A8B"
|
||||
"702D00000000624000001B84AADE5581141565EED165BA79999425204A8491C73B1301E34FE1E1E41100645698929E4419455BBB551BF09C08"
|
||||
"EEBEE19021E6B7E3D1D989968EF49970F10130E722000000012505A5B1C2558F56F16F29177518F3DD6CF827085D7B9E2806CD5EBDE810DD1F"
|
||||
"D7B40F710AC45898929E4419455BBB551BF09C08EEBEE19021E6B7E3D1D989968EF49970F101305A00080FA03A44668A2B96DFDE11BF0817CC"
|
||||
"6DF60C4E3508D43EB72AED00000051E1E1E411003756A2A1BC9A62AAEB2A2A70F895587A3FB752514AA03F8C6E7C84864653B8673E03E72200"
|
||||
"00000125059FB7B734000000000000003C3C000000000000000055172E9EC4F2ED6C22EDA53FE500AF1A1F5ACB0D14106842AD0F63D3CB235B"
|
||||
"EBBE5A00080FA03A44668A2B96DFDE11BF0817CC6DF60C4E3508D43EB72AED0000005161400000000007A1208214246B3E06AB367AB9614566"
|
||||
"B6F90C718B52A4440883141565EED165BA79999425204A8491C73B1301E34FE1E1E511006125059DE0B155C0F457C1104D45881194D53DBB40"
|
||||
"B81F9D812FC507637C4294527C06FDD69A3856A969AD4B14312DFA8739258A6A94C0868F1A3E7D1A1EF4839983521A4F6953FCE66240000000"
|
||||
"0011B75AE1E7220000000024046044C82D00000000202B00001836202C00000F3F62400000000012057A81143A44668A2B96DFDE11BF0817CC"
|
||||
"6DF60C4E3508D48914651B85AF14BE4F60AFBF5ADAA4F367061C2DA1D4E1E1E51100642505A5B1C2558F56F16F29177518F3DD6CF827085D7B"
|
||||
"9E2806CD5EBDE810DD1FD7B40F710AC456CE29B8547FC486F45704C1DE539B748587872AA803FB53FB938E282547DF2CFEE722000000003200"
|
||||
"0000000000004558DE4F51B35BE5A98D0C97BE07378E9CB56FFE3F861E544E0ABAC5ED765E2F781982140B639A808E3B9742A25334E5CF68EE"
|
||||
"DEDE52F54EE1E1E51100642505A4EA55556022061889BB4DA5242B7AB048C3D751ACEE5788ED4120B2E70F68E9A322A1E956D7A8E1C70CD8A0"
|
||||
"3AE9BAA41BC0898471FC26DA3712748A803B2F32007CDCB0DCE7220000000031000000000000003D32000000000000003B58B6629B8F178A18"
|
||||
"C2926F1ADA669262B5D362BFC2DB1417D837CBF382AA37F2D88214246B3E06AB367AB9614566B6F90C718B52A44408E1E1F1031000";
|
||||
|
||||
constinit auto const kHASH2 = "D7604B124D5D9C89EC1854A6CBD5A1FFD92502E945411B9C8DE397E7F19A74F8";
|
||||
|
||||
auto
|
||||
createTestData()
|
||||
{
|
||||
auto transactions = std::vector{
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN, kHASH2, kTXN_META2, kTXN_HEX2),
|
||||
util::createTransaction(ripple::TxType::ttAMM_CREATE), // not NFT - will be filtered
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN), // not unique - will be filtered
|
||||
};
|
||||
|
||||
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
|
||||
return etlng::model::LedgerData{
|
||||
.transactions = std::move(transactions),
|
||||
.objects = {},
|
||||
.successors = {},
|
||||
.edgeKeys = {},
|
||||
.header = header,
|
||||
.rawHeader = {},
|
||||
.seq = kSEQ
|
||||
};
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct NFTExtTests : util::prometheus::WithPrometheus, MockBackendTest {
|
||||
protected:
|
||||
etlng::impl::NFTExt ext_{backend_};
|
||||
};
|
||||
|
||||
TEST_F(NFTExtTests, OnLedgerDataFiltersAndWritesNFTs)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
|
||||
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
|
||||
});
|
||||
EXPECT_CALL(*backend_, writeNFTTransactions);
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(NFTExtTests, OnInitialDataFiltersAndWritesNFTs)
|
||||
{
|
||||
auto const data = createTestData();
|
||||
|
||||
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) {
|
||||
EXPECT_EQ(nfts.size(), 2); // AMM filtered out, two BURN txs are not unique
|
||||
});
|
||||
EXPECT_CALL(*backend_, writeNFTTransactions);
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(NFTExtTests, OnInitialObjectExtractsAndWritesNFTData)
|
||||
{
|
||||
auto const data = util::createObjectWithTwoNFTs();
|
||||
|
||||
EXPECT_CALL(*backend_, writeNFTs).WillOnce([](auto const& nfts) { EXPECT_EQ(nfts.size(), 2); });
|
||||
|
||||
ext_.onInitialObject(kSEQ, data);
|
||||
}
|
||||
@@ -1,639 +0,0 @@
|
||||
//------------------------------------------------------------------------------
|
||||
/*
|
||||
This file is part of clio: https://github.com/XRPLF/clio
|
||||
Copyright (c) 2025, the clio developers.
|
||||
|
||||
Permission to use, copy, modify, and distribute this software for any
|
||||
purpose with or without fee is hereby granted, provided that the above
|
||||
copyright notice and this permission notice appear in all copies.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
||||
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
||||
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
|
||||
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
||||
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
*/
|
||||
//==============================================================================
|
||||
|
||||
#include "data/DBHelpers.hpp"
|
||||
#include "data/Types.hpp"
|
||||
#include "etlng/Models.hpp"
|
||||
#include "etlng/impl/ext/Successor.hpp"
|
||||
#include "util/Assert.hpp"
|
||||
#include "util/BinaryTestObject.hpp"
|
||||
#include "util/MockAssert.hpp"
|
||||
#include "util/MockBackendTestFixture.hpp"
|
||||
#include "util/MockLedgerCache.hpp"
|
||||
#include "util/MockPrometheus.hpp"
|
||||
#include "util/StringUtils.hpp"
|
||||
#include "util/TestObject.hpp"
|
||||
|
||||
#include <gmock/gmock.h>
|
||||
#include <gtest/gtest.h>
|
||||
#include <xrpl/basics/base_uint.h>
|
||||
#include <xrpl/protocol/TxFormats.h>
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
#include <optional>
|
||||
#include <queue>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
using namespace etlng::impl;
|
||||
using namespace data;
|
||||
|
||||
namespace {
|
||||
constinit auto const kSEQ = 123u;
|
||||
constinit auto const kLEDGER_HASH = "4BC50C9B0D8515D3EAAE1E74B29A95804346C491EE1A95BF25E4AAB854A6A652";
|
||||
|
||||
auto
|
||||
createTestData(std::vector<etlng::model::Object> objects)
|
||||
{
|
||||
auto transactions = std::vector{
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_BURN),
|
||||
util::createTransaction(ripple::TxType::ttNFTOKEN_CREATE_OFFER),
|
||||
};
|
||||
|
||||
auto const header = createLedgerHeader(kLEDGER_HASH, kSEQ);
|
||||
return etlng::model::LedgerData{
|
||||
.transactions = std::move(transactions),
|
||||
.objects = std::move(objects),
|
||||
.successors = {},
|
||||
.edgeKeys = {},
|
||||
.header = header,
|
||||
.rawHeader = {},
|
||||
.seq = kSEQ
|
||||
};
|
||||
}
|
||||
|
||||
[[maybe_unused]] auto
|
||||
createInitialTestData(std::vector<ripple::uint256> edgeKeys)
|
||||
{
|
||||
// initial data expects objects to be empty as well as non-empty edgeKeys
|
||||
ASSERT(not edgeKeys.empty(), "Initial data requires edgeKeys");
|
||||
|
||||
auto ret = createTestData({});
|
||||
ret.edgeKeys = std::make_optional<std::vector<std::string>>();
|
||||
std::ranges::transform(edgeKeys, std::back_inserter(ret.edgeKeys.value()), &uint256ToString);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
struct SuccessorExtTests : util::prometheus::WithPrometheus, MockBackendTest {
|
||||
protected:
|
||||
MockLedgerCache cache_;
|
||||
etlng::impl::SuccessorExt ext_{backend_, cache_};
|
||||
};
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsNotFullButSuccessorsNotPresent)
|
||||
{
|
||||
auto const data = createTestData({});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(false));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataLogicErrorIfCacheIsFullButLatestSeqDiffersAndSuccessorsNotPresent)
|
||||
{
|
||||
auto const data = createTestData({});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ - 1));
|
||||
|
||||
EXPECT_THROW(ext_.onLedgerData(data), std::logic_error);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
|
||||
auto const data = createTestData({
|
||||
deletedObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(Blob{'0'}));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorAndNoBookBase)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
|
||||
auto const data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBase)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
|
||||
auto const data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const bookBase = getBookBase(createdObj.key);
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillRepeatedly(testing::Return(LedgerObject{}));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(
|
||||
SuccessorExtTests,
|
||||
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndMatchingSuccessorInCache
|
||||
)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
|
||||
auto const data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const bookBase = getBookBase(createdObj.key);
|
||||
|
||||
[[maybe_unused]] testing::InSequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(data::Blob{'0'}));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
|
||||
.WillRepeatedly(testing::Return(LedgerObject{.key = createdObj.key, .blob = {}}));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(
|
||||
SuccessorExtTests,
|
||||
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseButNoCurrentObjAndNoSuccessorInCache
|
||||
)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
|
||||
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
|
||||
auto const data = createTestData({
|
||||
deletedObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const bookBase = getBookBase(deletedObj.key);
|
||||
auto const oldCachedObj = createdObj.data;
|
||||
|
||||
[[maybe_unused]] testing::InSequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
|
||||
|
||||
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(
|
||||
SuccessorExtTests,
|
||||
OnLedgerDataWithDeletedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndCurrentObjAndSuccessorInCache
|
||||
)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
|
||||
auto const deletedObj = util::createObjectWithBookBase(Object::ModType::Deleted, objKey);
|
||||
auto const data = createTestData({
|
||||
deletedObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const bookBase = getBookBase(deletedObj.key);
|
||||
auto const oldCachedObj = createdObj.data;
|
||||
|
||||
[[maybe_unused]] testing::InSequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillOnce(testing::Return(oldCachedObj));
|
||||
|
||||
EXPECT_CALL(cache_, get(deletedObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
|
||||
.WillRepeatedly(testing::Return(LedgerObject{.key = deletedObj.key, .blob = {}}));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, uint256ToString(deletedObj.key)));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndWithCachedPredecessorAndSuccessor)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const predKey =
|
||||
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C"
|
||||
));
|
||||
auto const succKey =
|
||||
binaryStringToUint256(hexStringToBinaryString("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
|
||||
));
|
||||
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
|
||||
auto const data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ))
|
||||
.WillOnce(testing::Return(data::LedgerObject{.key = predKey, .blob = {}}));
|
||||
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ))
|
||||
.WillOnce(testing::Return(data::LedgerObject{.key = succKey, .blob = {}}));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(predKey), kSEQ, uint256ToString(createdObj.key)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(succKey)));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithCreatedObjectAndIncludedSuccessors)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObject(Object::ModType::Created, objKey);
|
||||
auto data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const succ = util::createSuccessor();
|
||||
data.successors = {succ, succ, succ};
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, auto{succ.firstBook}))
|
||||
.Times(data.successors->size());
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.predecessor}, kSEQ, auto{createdObj.keyRaw}));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{createdObj.keyRaw}, kSEQ, auto{createdObj.successor}));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnLedgerDataWithDeletedObjectAndIncludedSuccessorsWithoutFirstBook)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
|
||||
auto data = createTestData({
|
||||
deletedObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto succ = util::createSuccessor();
|
||||
succ.firstBook = {}; // empty will be transformed into kLAST_KEY
|
||||
data.successors = {succ, succ};
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{succ.bookBase}, kSEQ, uint256ToString(data::kLAST_KEY)))
|
||||
.Times(data.successors->size());
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{deletedObj.predecessor}, kSEQ, auto{deletedObj.successor}));
|
||||
|
||||
ext_.onLedgerData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndNoSuccessorsForEdgeKeys)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
|
||||
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
|
||||
auto const data = createInitialTestData({firstKey, secondKey});
|
||||
|
||||
auto successorChain = std::queue<ripple::uint256>();
|
||||
successorChain.push(firstKey);
|
||||
successorChain.push(secondKey);
|
||||
|
||||
[[maybe_unused]] testing::Sequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
|
||||
|
||||
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
|
||||
.Times(3)
|
||||
.InSequence(inSeq)
|
||||
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
|
||||
if (successorChain.empty())
|
||||
return std::nullopt;
|
||||
|
||||
auto v = successorChain.front();
|
||||
successorChain.pop();
|
||||
return data::LedgerObject{.key = v, .blob = {'0'}};
|
||||
});
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
for (auto const& key : data.edgeKeys.value()) {
|
||||
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
|
||||
.InSequence(inSeq)
|
||||
.WillOnce(testing::Return(std::nullopt));
|
||||
}
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsButNotBookDirAndSuccessorsForEdgeKeys)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
|
||||
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
|
||||
auto const data = createInitialTestData({firstKey, secondKey});
|
||||
|
||||
auto successorChain = std::queue<ripple::uint256>();
|
||||
successorChain.push(firstKey);
|
||||
successorChain.push(secondKey);
|
||||
|
||||
[[maybe_unused]] testing::Sequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
|
||||
|
||||
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
|
||||
.Times(3)
|
||||
.InSequence(inSeq)
|
||||
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
|
||||
if (successorChain.empty())
|
||||
return std::nullopt;
|
||||
|
||||
auto v = successorChain.front();
|
||||
successorChain.pop();
|
||||
return data::LedgerObject{.key = v, .blob = {'0'}};
|
||||
});
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
for (auto const& key : data.edgeKeys.value()) {
|
||||
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
|
||||
.InSequence(inSeq)
|
||||
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {}}));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey)));
|
||||
}
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnInitialDataWithSuccessorsAndBookDirAndSuccessorsForEdgeKeys)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
|
||||
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
|
||||
auto const data = createInitialTestData({firstKey, secondKey});
|
||||
|
||||
auto successorChain = std::queue<ripple::uint256>();
|
||||
successorChain.push(firstKey);
|
||||
successorChain.push(secondKey);
|
||||
|
||||
auto const bookBaseObj = util::createObjectWithBookBase(Object::ModType::Created);
|
||||
auto const bookBase = getBookBase(bookBaseObj.key);
|
||||
|
||||
[[maybe_unused]] testing::Sequence const inSeq;
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
|
||||
|
||||
EXPECT_CALL(cache_, getSuccessor(testing::_, kSEQ))
|
||||
.Times(3)
|
||||
.InSequence(inSeq)
|
||||
.WillRepeatedly([&](auto&&, auto&&) -> std::optional<data::LedgerObject> {
|
||||
if (successorChain.empty())
|
||||
return std::nullopt;
|
||||
|
||||
auto v = successorChain.front();
|
||||
successorChain.pop();
|
||||
return data::LedgerObject{.key = v, .blob = bookBaseObj.data};
|
||||
});
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(firstKey)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(secondKey), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
EXPECT_CALL(cache_, get(bookBase, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ))
|
||||
.WillRepeatedly(testing::Return(data::LedgerObject{.key = firstKey, .blob = data::Blob{'1'}}));
|
||||
EXPECT_CALL(
|
||||
*backend_, writeSuccessor(uint256ToString(bookBase), kSEQ, testing::_)
|
||||
); // Called once because firstKey returned repeatedly above
|
||||
|
||||
for (auto const& key : data.edgeKeys.value()) {
|
||||
EXPECT_CALL(cache_, getSuccessor(*ripple::uint256::fromVoidChecked(key), kSEQ))
|
||||
.InSequence(inSeq)
|
||||
.WillOnce(testing::Return(data::LedgerObject{.key = firstKey, .blob = {'1'}}));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(auto{key}, kSEQ, uint256ToString(firstKey))).InSequence(inSeq);
|
||||
}
|
||||
|
||||
ext_.onInitialData(data);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnInitialObjectsWithEmptyLastKey)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const lastKey = std::string{};
|
||||
auto const data = std::vector{
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
|
||||
),
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
|
||||
),
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
|
||||
),
|
||||
};
|
||||
|
||||
std::string lk = lastKey;
|
||||
for (auto const& obj : data) {
|
||||
if (not lk.empty())
|
||||
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
|
||||
lk = uint256ToString(obj.key);
|
||||
}
|
||||
|
||||
ext_.onInitialObjects(kSEQ, data, lastKey);
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtTests, OnInitialObjectsWithNonEmptyLastKey)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const lastKey =
|
||||
uint256ToString(ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D"));
|
||||
auto const data = std::vector{
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E"
|
||||
),
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960F"
|
||||
),
|
||||
util::createObject(
|
||||
Object::ModType::Created, "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E89610"
|
||||
),
|
||||
};
|
||||
|
||||
std::string lk = lastKey;
|
||||
for (auto const& obj : data) {
|
||||
EXPECT_CALL(*backend_, writeSuccessor(std::move(lk), kSEQ, uint256ToString(obj.key)));
|
||||
lk = uint256ToString(obj.key);
|
||||
}
|
||||
|
||||
ext_.onInitialObjects(kSEQ, data, lastKey);
|
||||
}
|
||||
|
||||
struct SuccessorExtAssertTests : common::util::WithMockAssert, SuccessorExtTests {};
|
||||
|
||||
TEST_F(SuccessorExtAssertTests, OnLedgerDataWithDeletedObjectAssertsIfGetDeletedIsNotInCache)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const deletedObj = util::createObject(Object::ModType::Deleted, objKey);
|
||||
auto const data = createTestData({
|
||||
deletedObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(deletedObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
EXPECT_CALL(cache_, getDeleted(deletedObj.key, kSEQ - 1)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
|
||||
}
|
||||
|
||||
TEST_F(
|
||||
SuccessorExtAssertTests,
|
||||
OnLedgerDataWithCreatedObjectButWithoutCachedPredecessorAndSuccessorWithBookBaseAndBookSuccessorNotInCache
|
||||
)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const objKey = "B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960D";
|
||||
auto const createdObj = util::createObjectWithBookBase(Object::ModType::Created, objKey);
|
||||
auto const data = createTestData({
|
||||
createdObj,
|
||||
util::createObject(Object::ModType::Modified),
|
||||
});
|
||||
auto const bookBase = getBookBase(createdObj.key);
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillRepeatedly(testing::Return(true));
|
||||
EXPECT_CALL(cache_, latestLedgerSequence()).WillRepeatedly(testing::Return(kSEQ));
|
||||
|
||||
EXPECT_CALL(cache_, getPredecessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
EXPECT_CALL(cache_, getSuccessor(createdObj.key, kSEQ)).WillRepeatedly(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(data::kFIRST_KEY), kSEQ, uint256ToString(createdObj.key)));
|
||||
EXPECT_CALL(*backend_, writeSuccessor(uint256ToString(createdObj.key), kSEQ, uint256ToString(data::kLAST_KEY)));
|
||||
|
||||
EXPECT_CALL(cache_, get(createdObj.key, kSEQ)).WillOnce(testing::Return(data::Blob{'0'}));
|
||||
EXPECT_CALL(cache_, getSuccessor(bookBase, kSEQ)).WillOnce(testing::Return(std::nullopt));
|
||||
|
||||
EXPECT_CLIO_ASSERT_FAIL({ ext_.onLedgerData(data); });
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtAssertTests, OnInitialDataNotIsFull)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const data = createTestData({
|
||||
util::createObject(Object::ModType::Modified),
|
||||
util::createObject(Object::ModType::Created),
|
||||
});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(false));
|
||||
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullButNoEdgeKeys)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto data = createTestData({});
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
|
||||
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
|
||||
}
|
||||
|
||||
TEST_F(SuccessorExtAssertTests, OnInitialDataIsFullWithEdgeKeysButHasObjects)
|
||||
{
|
||||
using namespace etlng::model;
|
||||
|
||||
auto const firstKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960C");
|
||||
auto const secondKey = ripple::uint256("B00AA769C00726371689ED66A7CF57C2502F1BF4BDFF2ACADF67A2A7B5E8960E");
|
||||
auto data = createInitialTestData({firstKey, secondKey});
|
||||
data.objects = {util::createObject()};
|
||||
|
||||
EXPECT_CALL(cache_, isFull()).WillOnce(testing::Return(true));
|
||||
EXPECT_CLIO_ASSERT_FAIL({ ext_.onInitialData(data); });
|
||||
}
|
||||
@@ -419,8 +419,7 @@ TEST_F(RPCHelpersTest, DeliverMaxAliasV1)
|
||||
for (size_t i = 0; i < inputArray.size(); i++) {
|
||||
auto req = boost::json::parse(inputArray[i]).as_object();
|
||||
insertDeliverMaxAlias(req, 1);
|
||||
auto const expectedReq = boost::json::parse(outputArray[i]).as_object();
|
||||
EXPECT_EQ(req, expectedReq) << req << "\n" << expectedReq;
|
||||
EXPECT_EQ(req, boost::json::parse(outputArray[i]).as_object());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -41,9 +41,9 @@
|
||||
|
||||
using namespace rpc;
|
||||
using namespace data;
|
||||
using namespace testing;
|
||||
|
||||
namespace json = boost::json;
|
||||
using namespace testing;
|
||||
|
||||
namespace {
|
||||
|
||||
|
||||
@@ -522,7 +522,9 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest)
|
||||
"limit": "100",
|
||||
"limit_peer": "200",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
},
|
||||
{
|
||||
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
|
||||
@@ -531,7 +533,9 @@ TEST_F(RPCAccountLinesHandlerTest, DefaultParameterTest)
|
||||
"limit": "200",
|
||||
"limit_peer": "100",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
}
|
||||
]
|
||||
})";
|
||||
@@ -733,6 +737,7 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField)
|
||||
"limit_peer": "200",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": true,
|
||||
"peer_authorized": true,
|
||||
"freeze_peer": true,
|
||||
@@ -747,6 +752,7 @@ TEST_F(RPCAccountLinesHandlerTest, OptionalResponseField)
|
||||
"quality_in": 0,
|
||||
"quality_out": 0,
|
||||
"no_ripple": true,
|
||||
"no_ripple_peer": false,
|
||||
"authorized": true,
|
||||
"freeze": true,
|
||||
"deep_freeze": true
|
||||
@@ -986,7 +992,9 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin)
|
||||
"limit": "100",
|
||||
"limit_peer": "200",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
}},
|
||||
{{
|
||||
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
|
||||
@@ -995,7 +1003,9 @@ TEST_F(RPCAccountLinesHandlerTest, LimitLessThanMin)
|
||||
"limit": "200",
|
||||
"limit_peer": "100",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
}}
|
||||
]
|
||||
}})",
|
||||
@@ -1063,7 +1073,9 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax)
|
||||
"limit": "100",
|
||||
"limit_peer": "200",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
}},
|
||||
{{
|
||||
"account": "rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun",
|
||||
@@ -1072,7 +1084,9 @@ TEST_F(RPCAccountLinesHandlerTest, LimitMoreThanMax)
|
||||
"limit": "200",
|
||||
"limit_peer": "100",
|
||||
"quality_in": 0,
|
||||
"quality_out": 0
|
||||
"quality_out": 0,
|
||||
"no_ripple": false,
|
||||
"no_ripple_peer": false
|
||||
}}
|
||||
]
|
||||
}})",
|
||||
|
||||
@@ -31,6 +31,7 @@
|
||||
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/asio/ssl/context.hpp>
|
||||
#include <boost/beast/core/flat_buffer.hpp>
|
||||
#include <boost/beast/http/field.hpp>
|
||||
#include <boost/beast/http/message.hpp>
|
||||
@@ -299,7 +300,8 @@ TEST_F(HttpConnectionTests, Upgrade)
|
||||
[&]() { ASSERT_TRUE(expectedResult.has_value()) << expectedResult.error().message(); }();
|
||||
[&]() { ASSERT_TRUE(expectedResult.value()); }();
|
||||
|
||||
auto expectedWsConnection = connection.upgrade(tagDecoratorFactory_, yield);
|
||||
std::optional<boost::asio::ssl::context> sslContext;
|
||||
auto expectedWsConnection = connection.upgrade(sslContext, tagDecoratorFactory_, yield);
|
||||
[&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/post.hpp>
|
||||
#include <boost/asio/spawn.hpp>
|
||||
#include <boost/asio/ssl/context.hpp>
|
||||
#include <boost/asio/steady_timer.hpp>
|
||||
#include <boost/beast/core/flat_buffer.hpp>
|
||||
#include <boost/beast/http/status.hpp>
|
||||
@@ -73,7 +74,8 @@ struct WebWsConnectionTests : SyncAsioContextTest {
|
||||
ASSERT_TRUE(expectedTrue.value()) << "Expected upgrade request";
|
||||
}();
|
||||
|
||||
auto expectedWsConnection = httpConnection.upgrade(tagDecoratorFactory_, yield);
|
||||
std::optional<boost::asio::ssl::context> sslContext;
|
||||
auto expectedWsConnection = httpConnection.upgrade(sslContext, tagDecoratorFactory_, yield);
|
||||
[&]() { ASSERT_TRUE(expectedWsConnection.has_value()) << expectedWsConnection.error().message(); }();
|
||||
auto connection = std::move(expectedWsConnection).value();
|
||||
auto wsConnectionPtr = dynamic_cast<PlainWsConnection*>(connection.release());
|
||||
|
||||
@@ -6,7 +6,6 @@ set(GO_SOURCE_DIR "${CMAKE_SOURCE_DIR}/tools/snapshot")
|
||||
set(PROTO_INC_DIR "${xrpl_PACKAGE_FOLDER_RELEASE}/include/xrpl/proto")
|
||||
set(PROTO_SOURCE_DIR "${PROTO_INC_DIR}/org/xrpl/rpc/v1/")
|
||||
set(GO_OUTPUT "${CMAKE_BINARY_DIR}/clio_snapshot")
|
||||
file(GLOB_RECURSE GO_SOURCES ${GO_SOURCE_DIR}/*.go)
|
||||
|
||||
set(PROTO_FILES
|
||||
${PROTO_SOURCE_DIR}/xrp_ledger.proto ${PROTO_SOURCE_DIR}/ledger.proto ${PROTO_SOURCE_DIR}/get_ledger.proto
|
||||
@@ -35,7 +34,7 @@ endforeach()
|
||||
foreach (proto ${PROTO_FILES})
|
||||
get_filename_component(proto_name ${proto} NAME_WE)
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go
|
||||
OUTPUT ${GO_SOURCE_DIR}/${proto_name}.pb.go
|
||||
COMMAND
|
||||
protoc ${GO_OPTS} ${GRPC_OPTS}
|
||||
--go-grpc_out=${GO_SOURCE_DIR} -I${PROTO_INC_DIR} ${proto} --plugin=${GOPATH_VALUE}/bin/protoc-gen-go
|
||||
@@ -45,7 +44,7 @@ foreach (proto ${PROTO_FILES})
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${GO_IMPORT_PATH}/${proto_name}.pb.go)
|
||||
list(APPEND GENERATED_GO_FILES ${GO_SOURCE_DIR}/${proto_name}.pb.go)
|
||||
endforeach ()
|
||||
|
||||
add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
|
||||
@@ -53,16 +52,15 @@ add_custom_target(build_clio_snapshot ALL DEPENDS run_go_tests ${GO_OUTPUT})
|
||||
add_custom_target(run_go_tests
|
||||
COMMAND go test ./...
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
DEPENDS ${GENERATED_GO_FILES}
|
||||
COMMENT "Running clio_snapshot unittests"
|
||||
VERBATIM
|
||||
DEPENDS ${GENERATED_GO_FILES}
|
||||
)
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${GO_OUTPUT}
|
||||
COMMAND ${GO_EXECUTABLE} build -o ${GO_OUTPUT} ${GO_SOURCE_DIR}
|
||||
WORKING_DIRECTORY ${GO_SOURCE_DIR}
|
||||
DEPENDS ${GO_SOURCES}
|
||||
COMMENT "Building clio_snapshot"
|
||||
VERBATIM
|
||||
)
|
||||
|
||||
@@ -6,7 +6,6 @@ toolchain go1.22.11
|
||||
|
||||
require (
|
||||
github.com/golang/mock v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/spf13/pflag v1.0.6
|
||||
github.com/stretchr/testify v1.10.0
|
||||
google.golang.org/grpc v1.69.4
|
||||
|
||||
@@ -12,8 +12,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
|
||||
@@ -4,20 +4,20 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
grpcUser = "clio-snapshot"
|
||||
markerNum = 16
|
||||
maxConcurrency = 256
|
||||
firstAvailableLedger = 32570
|
||||
deltaDataFolderDiv = 10000
|
||||
grpcUser = "clio-snapshot"
|
||||
markerNum = 16
|
||||
)
|
||||
|
||||
type gRPCClient struct {
|
||||
@@ -44,25 +44,7 @@ func createGRPCClient(serverAddr string) (*gRPCClient, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
func getLedgerDeltaDataInParallel(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, ledgersHouse *ledgers.LedgersHouse) {
|
||||
sem := make(chan struct{}, maxConcurrency)
|
||||
var wg sync.WaitGroup
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
wg.Add(1)
|
||||
sem <- struct{}{}
|
||||
|
||||
go func(seq uint32) {
|
||||
defer wg.Done()
|
||||
log.Printf("Process delta sequence: %d\n", seq)
|
||||
getLedgerDeltaData(client, seq, ledgersHouse)
|
||||
<-sem
|
||||
}(i)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgersHouse *ledgers.LedgersHouse) {
|
||||
func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -74,27 +56,49 @@ func getLedgerDeltaData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgers
|
||||
}
|
||||
request.Ledger = ledger
|
||||
request.User = grpcUser
|
||||
request.GetObjectNeighbors = true
|
||||
request.Transactions = true
|
||||
request.Expand = true
|
||||
|
||||
// The first available ledger doesn't have diff data
|
||||
request.GetObjects = firstAvailableLedger != seq
|
||||
request.GetObjectNeighbors = firstAvailableLedger != seq
|
||||
request.GetObjects = true
|
||||
|
||||
response, err := client.GetLedger(ctx, &request)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger delta data: %v - seq: %d", err, seq)
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
}
|
||||
|
||||
err = ledgersHouse.WriteLedgerDeltaData(seq, response)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing ledger delta data: %v", err)
|
||||
}
|
||||
saveLedgerDeltaData(seq, response, path)
|
||||
|
||||
log.Printf("Processing delta sequence: %d\n", seq)
|
||||
}
|
||||
|
||||
func roundDown(n uint32, roundTo uint32) uint32 {
|
||||
if roundTo == 0 {
|
||||
return n
|
||||
}
|
||||
return n - (n % roundTo)
|
||||
}
|
||||
|
||||
func saveLedgerDeltaData(seq uint32, response *pb.GetLedgerResponse, path string) {
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
protoData, err := proto.Marshal(response)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
|
||||
|
||||
err = os.WriteFile(filePath, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func generateMarkers(markerNum uint32) [][32]byte {
|
||||
var byteArray [32]byte
|
||||
|
||||
@@ -110,7 +114,19 @@ func generateMarkers(markerNum uint32) [][32]byte {
|
||||
return byteArrayList
|
||||
}
|
||||
|
||||
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, ledgerHouse *ledgers.LedgersHouse) {
|
||||
func saveLedgerData(path string, data *pb.GetLedgerDataResponse) {
|
||||
protoData, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
log.Fatalf("Error marshalling data: %v", err)
|
||||
}
|
||||
|
||||
err = os.WriteFile(path, protoData, 0644)
|
||||
if err != nil {
|
||||
log.Fatalf("failed to write file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byte, end []byte, path string) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -127,22 +143,25 @@ func getLedgerData(client pb.XRPLedgerAPIServiceClient, seq uint32, marker []byt
|
||||
}
|
||||
request.User = grpcUser
|
||||
|
||||
subPath := filepath.Join(path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%x", marker))
|
||||
err := os.MkdirAll(subPath, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
|
||||
for request.Marker != nil {
|
||||
res, err := client.GetLedgerData(ctx, &request)
|
||||
if err != nil {
|
||||
log.Fatalf("Error getting ledger data: %v", err)
|
||||
}
|
||||
|
||||
err = ledgerHouse.WriteLedgerData(seq, request.Marker, res)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing ledger data: %v", err)
|
||||
}
|
||||
log.Printf("Saving ledger data %x", request.Marker)
|
||||
filePath := filepath.Join(subPath, fmt.Sprintf("%x.dat", request.Marker))
|
||||
saveLedgerData(filePath, res)
|
||||
request.Marker = res.Marker
|
||||
}
|
||||
}
|
||||
|
||||
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHouse *ledgers.LedgersHouse) {
|
||||
func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, path string) {
|
||||
log.Printf("Processing full sequence: %d\n", seq)
|
||||
|
||||
markers := generateMarkers(markerNum)
|
||||
@@ -157,9 +176,11 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHo
|
||||
end = markers[i+1][:]
|
||||
}
|
||||
|
||||
fmt.Printf("Got ledger data marker: %x-%x\n", marker, end)
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
getLedgerData(client, seq, marker[:], end, ledgerHouse)
|
||||
getLedgerData(client, seq, marker[:], end, path)
|
||||
}()
|
||||
|
||||
}
|
||||
@@ -167,7 +188,19 @@ func getLedgerFullData(client pb.XRPLedgerAPIServiceClient, seq uint32, ledgerHo
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func checkPath(path string) {
|
||||
if _, err := os.Stat(path); os.IsNotExist(err) {
|
||||
// Create the directory if it doesn't exist
|
||||
err := os.MkdirAll(path, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating directory: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
@@ -179,21 +212,20 @@ func ExportFromFullLedger(grpcServer string, startSeq uint32, endSeq uint32, pat
|
||||
}
|
||||
|
||||
func exportFromFullLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
getLedgerFullData(client, startSeq, ledgersHouse)
|
||||
getLedgerFullData(client, startSeq, path)
|
||||
|
||||
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
|
||||
|
||||
err := ledgersHouse.SetRange(startSeq, endSeq)
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing range: %v", err)
|
||||
//We need to fetch the ledger header and txs for startSeq as well
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from full ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
}
|
||||
|
||||
func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, path string) {
|
||||
checkPath(path)
|
||||
|
||||
client, err := createGRPCClient(grpcServer)
|
||||
if err != nil {
|
||||
log.Fatalf("Error creating gRPC client: %v", err)
|
||||
@@ -205,27 +237,8 @@ func ExportFromDeltaLedger(grpcServer string, startSeq uint32, endSeq uint32, pa
|
||||
}
|
||||
|
||||
func exportFromDeltaLedgerImpl(client pb.XRPLedgerAPIServiceClient, startSeq uint32, endSeq uint32, path string) {
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
_, oldEnd, err := ledgersHouse.GetRange()
|
||||
if err != nil {
|
||||
log.Fatalf("Can't find existing snapshot to extend: %v", err)
|
||||
}
|
||||
|
||||
if oldEnd < startSeq-1 {
|
||||
log.Fatalf("Missing delta ledger from %d to %d", oldEnd, startSeq)
|
||||
}
|
||||
|
||||
if oldEnd >= endSeq {
|
||||
log.Fatalf("The snapshot already contains the requested delta ledger")
|
||||
}
|
||||
|
||||
getLedgerDeltaDataInParallel(client, startSeq, endSeq, ledgersHouse)
|
||||
|
||||
err = ledgersHouse.AppendDeltaLedger(startSeq, endSeq)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Error writing new range: %v", err)
|
||||
for i := startSeq; i <= endSeq; i++ {
|
||||
getLedgerDeltaData(client, i, path)
|
||||
}
|
||||
|
||||
log.Printf("Exporting from ledger: %d to %d at path %s\n", startSeq, endSeq, path)
|
||||
|
||||
@@ -1,48 +1,26 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
"xrplf/clio/clio_snapshot/mocks"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
)
|
||||
|
||||
// Matcher used to verify the GetLedgerRequest parameters
|
||||
type LedgerRequestMatcher struct {
|
||||
expectedObjects bool
|
||||
expectedNeighbors bool
|
||||
}
|
||||
|
||||
func (m LedgerRequestMatcher) Matches(x interface{}) bool {
|
||||
req, ok := x.(*pb.GetLedgerRequest)
|
||||
return ok && req.GetObjects == m.expectedObjects && req.GetObjectNeighbors == m.expectedNeighbors
|
||||
}
|
||||
|
||||
func (m LedgerRequestMatcher) String() string {
|
||||
return fmt.Sprintf("LedgerRequest with objects=%v neighbors=%v", m.expectedObjects, m.expectedNeighbors)
|
||||
}
|
||||
|
||||
func matchObjectsEquals(objects bool, neighbors bool) gomock.Matcher {
|
||||
return LedgerRequestMatcher{objects, neighbors}
|
||||
}
|
||||
|
||||
func TestExportDeltaLedgerData(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
startSeq uint32
|
||||
endSeq uint32
|
||||
}{
|
||||
{"OneSeq", 700000, 700000},
|
||||
{"MultipleSeq", 700000, 700019},
|
||||
{"FirstAvailableLedger", firstAvailableLedger, firstAvailableLedger},
|
||||
{"FirstAvailableLedgerMultipleSeq", firstAvailableLedger, firstAvailableLedger + 2},
|
||||
{"OneSeq", 1, 1},
|
||||
{"MultipleSeq", 1, 20},
|
||||
{"EndSeqLessThanStartSeq", 20, 1},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -59,26 +37,15 @@ func TestExportDeltaLedgerData(t *testing.T) {
|
||||
times = 0
|
||||
}
|
||||
|
||||
if tt.startSeq == firstAvailableLedger {
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(false, false)).Return(mockResponse, nil).Times(1)
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times) - 1)
|
||||
} else {
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), matchObjectsEquals(true, true)).Return(mockResponse, nil).Times(int(times))
|
||||
}
|
||||
|
||||
os.MkdirAll("test", os.ModePerm)
|
||||
|
||||
manifest := ledgers.NewManifest("test")
|
||||
manifest.SetLedgerRange(tt.startSeq-1, tt.startSeq-1)
|
||||
mockClient.EXPECT().GetLedger(gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(int(times))
|
||||
|
||||
defer os.RemoveAll("test")
|
||||
|
||||
exportFromDeltaLedgerImpl(mockClient, tt.startSeq, tt.endSeq, "test")
|
||||
|
||||
seq1, seq2, err := manifest.Read()
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, tt.startSeq-1, seq1)
|
||||
assert.Equal(t, tt.endSeq, seq2)
|
||||
_, err := os.Stat("test")
|
||||
|
||||
assert.Equal(t, os.IsNotExist(err), tt.endSeq < tt.startSeq)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -125,6 +92,26 @@ func TestExportFullLedgerData(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoundDown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in1 uint32
|
||||
in2 uint32
|
||||
out uint32
|
||||
}{
|
||||
{"RoundDownToZero", 10, 0, 10},
|
||||
{"RoundDown12To10", 12, 10, 10},
|
||||
{"RoundDownToOne", 13, 1, 13},
|
||||
{"RoundDown100", 103, 100, 100},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGenerateMarkers(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
@@ -145,3 +132,22 @@ func TestGenerateMarkers(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"Path", "test"},
|
||||
{"NestedPath", "test/test"}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
checkPath(tt.path)
|
||||
defer os.RemoveAll(tt.path)
|
||||
_, err := os.Stat(tt.path)
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
const deltaDataFolderDiv = 10000
|
||||
const readWritePerm = 0644
|
||||
|
||||
func convertInnerMarkerToMarker(in []byte) []byte {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := make([]byte, len(in))
|
||||
out[0] = in[0] & 0xf0
|
||||
return out
|
||||
}
|
||||
|
||||
func checkPath(path string) error {
|
||||
dir := filepath.Dir(path)
|
||||
if _, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
// Create the directory if it doesn't exist
|
||||
err := os.MkdirAll(dir, os.ModePerm)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error creating directory: %s,%v", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func roundDown(n uint32, roundTo uint32) uint32 {
|
||||
if roundTo == 0 {
|
||||
return n
|
||||
}
|
||||
return n - (n % roundTo)
|
||||
}
|
||||
|
||||
type LedgersHouse struct {
|
||||
path string
|
||||
manifest *Manifest
|
||||
}
|
||||
|
||||
func NewLedgersHouse(path string) *LedgersHouse {
|
||||
return &LedgersHouse{path: path, manifest: NewManifest(path)}
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) deltaDataPath(seq uint32) string {
|
||||
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_diff_%d", roundDown(seq, deltaDataFolderDiv)))
|
||||
return filepath.Join(subPath, fmt.Sprintf("%d.dat", seq))
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) fullDataPath(seq uint32, marker string, innerMarker string) string {
|
||||
subPath := filepath.Join(lh.path, fmt.Sprintf("ledger_data_%d", seq), fmt.Sprintf("marker_%s", marker))
|
||||
return filepath.Join(subPath, fmt.Sprintf("%s.dat", innerMarker))
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) ReadLedgerDeltaData(seq uint32) (*pb.GetLedgerResponse, error) {
|
||||
path := lh.deltaDataPath(seq)
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ledger pb.GetLedgerResponse
|
||||
if err := proto.Unmarshal(data, &ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ledger, nil
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) WriteLedgerDeltaData(seq uint32, data *pb.GetLedgerResponse) error {
|
||||
path := lh.deltaDataPath(seq)
|
||||
err := checkPath(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataBytes, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, dataBytes, readWritePerm)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) ReadLedgerData(seq uint32, innerMarker []byte) (*pb.GetLedgerDataResponse, error) {
|
||||
marker := convertInnerMarkerToMarker(innerMarker)
|
||||
path := lh.fullDataPath(seq, fmt.Sprintf("%x", marker), fmt.Sprintf("%x", innerMarker))
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ledger pb.GetLedgerDataResponse
|
||||
if err := proto.Unmarshal(data, &ledger); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ledger, nil
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) WriteLedgerData(seq uint32, innerMarker []byte, data *pb.GetLedgerDataResponse) error {
|
||||
path := lh.fullDataPath(seq, fmt.Sprintf("%x", convertInnerMarkerToMarker(innerMarker)), fmt.Sprintf("%x", innerMarker))
|
||||
err := checkPath(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dataBytes, err := proto.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, dataBytes, readWritePerm)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) SetRange(startSeq uint32, endSeq uint32) error {
|
||||
return lh.manifest.SetLedgerRange(startSeq, endSeq)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) AppendDeltaLedger(startSeq uint32, endSeq uint32) error {
|
||||
return lh.manifest.AppendDeltaLedger(startSeq, endSeq)
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) IsExist() bool {
|
||||
return lh.manifest.IsExist()
|
||||
}
|
||||
|
||||
func (lh *LedgersHouse) GetRange() (uint32, uint32, error) {
|
||||
return lh.manifest.Read()
|
||||
}
|
||||
@@ -1,188 +0,0 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestCheckPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"Path", "test/d.dat"},
|
||||
{"NestedPath", "test/test/d.dat"}}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := checkPath(tt.path)
|
||||
assert.NoError(t, err)
|
||||
dir := filepath.Dir(tt.path)
|
||||
defer os.RemoveAll("test")
|
||||
_, err = os.Stat(dir)
|
||||
assert.False(t, os.IsNotExist(err))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestRoundDown(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
in1 uint32
|
||||
in2 uint32
|
||||
out uint32
|
||||
}{
|
||||
{"RoundDownToZero", 10, 0, 10},
|
||||
{"RoundDown12To10", 12, 10, 10},
|
||||
{"RoundDownToOne", 13, 1, 13},
|
||||
{"RoundDown100", 103, 100, 100},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, roundDown(tt.in1, tt.in2), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConvertInnerMarkerToMarker(t *testing.T) {
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
in []byte
|
||||
out []byte
|
||||
}{
|
||||
{"SingleByte", []byte{0x01}, []byte{0x00}},
|
||||
{"MultipleBytes", []byte{0x01, 0x02, 0x03}, []byte{0x00, 0x00, 0x00}},
|
||||
{"MultipleBytes2", []byte{0xf1, 0x02, 0x03}, []byte{0xf0, 0x00, 0x00}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
assert.Equal(t, convertInnerMarkerToMarker(tt.in), tt.out)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLedgersHouseGetDeltaPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("testdata")
|
||||
assert.Equal(t, lh.deltaDataPath(12345), "testdata/ledger_diff_10000/12345.dat")
|
||||
|
||||
assert.Equal(t, lh.deltaDataPath(3), "testdata/ledger_diff_0/3.dat")
|
||||
|
||||
assert.Equal(t, lh.deltaDataPath(0), "testdata/ledger_diff_0/0.dat")
|
||||
}
|
||||
|
||||
func TestLedgersHouseGetFullDataPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("testdata")
|
||||
assert.Equal(t, lh.fullDataPath(12345, "fffff", "ababab"), "testdata/ledger_data_12345/marker_fffff/ababab.dat")
|
||||
}
|
||||
|
||||
func TestLedgerHouseLedgerDeltaData(t *testing.T) {
|
||||
defer os.RemoveAll("testdata")
|
||||
lh := NewLedgersHouse("testdata")
|
||||
data, err := lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
|
||||
data, err = lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data != nil)
|
||||
assert.True(t, err == nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseInvalidLedgerDeltaPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("/etc")
|
||||
data, err := lh.ReadLedgerDeltaData(12345)
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
err = lh.WriteLedgerDeltaData(12345, &pb.GetLedgerResponse{})
|
||||
assert.True(t, err != nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseLedgerData(t *testing.T) {
|
||||
defer os.RemoveAll("testdata")
|
||||
lh := NewLedgersHouse("testdata")
|
||||
data, err := lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
|
||||
data, err = lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data != nil)
|
||||
assert.True(t, err == nil)
|
||||
}
|
||||
|
||||
func TestLedgerHouseInvalidLedgerDataPath(t *testing.T) {
|
||||
lh := NewLedgersHouse("/etc")
|
||||
data, err := lh.ReadLedgerData(12345, []byte{0x01})
|
||||
assert.True(t, data == nil)
|
||||
assert.True(t, err != nil)
|
||||
|
||||
err = lh.WriteLedgerData(12345, []byte{0x01}, &pb.GetLedgerDataResponse{})
|
||||
assert.True(t, err != nil)
|
||||
}
|
||||
|
||||
func TestLedgersHouseManifest(t *testing.T) {
|
||||
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
lh := NewLedgersHouse("testdata")
|
||||
startSeq, endSeq, err := lh.GetRange()
|
||||
assert.True(t, err != nil)
|
||||
assert.Equal(t, startSeq, uint32(0))
|
||||
assert.Equal(t, endSeq, uint32(0))
|
||||
assert.False(t, lh.IsExist())
|
||||
|
||||
lh.SetRange(1, 100)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(100))
|
||||
|
||||
lh.AppendDeltaLedger(100, 200)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(200))
|
||||
|
||||
lh.AppendDeltaLedger(201, 300)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(201, 100)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(302, 350)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
err = lh.AppendDeltaLedger(0, 350)
|
||||
assert.True(t, err != nil)
|
||||
assert.True(t, lh.IsExist())
|
||||
startSeq, endSeq, err = lh.GetRange()
|
||||
assert.True(t, err == nil)
|
||||
assert.Equal(t, startSeq, uint32(1))
|
||||
assert.Equal(t, endSeq, uint32(300))
|
||||
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
fileName = "manifest.txt"
|
||||
)
|
||||
|
||||
type Manifest struct {
|
||||
folderPath string
|
||||
filePath string
|
||||
}
|
||||
|
||||
func NewManifest(folderPath string) *Manifest {
|
||||
return &Manifest{
|
||||
folderPath: folderPath,
|
||||
filePath: filepath.Join(folderPath, fileName),
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *Manifest) SetLedgerRange(start uint32, end uint32) error {
|
||||
content := fmt.Sprintf("%d|%d", start, end)
|
||||
return fm.writeToFile(content)
|
||||
}
|
||||
|
||||
func (fm *Manifest) AppendDeltaLedger(delta1 uint32, delta2 uint32) error {
|
||||
start, end, err := fm.Read()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
//rewrite the range if new delta can extend the current range continuously
|
||||
if delta1 >= start && (end+1) >= delta1 && delta2 >= delta1 {
|
||||
return fm.SetLedgerRange(start, delta2)
|
||||
}
|
||||
|
||||
return fmt.Errorf("Invalid delta ledger range")
|
||||
}
|
||||
|
||||
func (fm *Manifest) writeToFile(content string) error {
|
||||
os.MkdirAll(fm.folderPath, os.ModePerm)
|
||||
file, err := os.OpenFile(fm.filePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.WriteString(content)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (fm *Manifest) IsExist() bool {
|
||||
_, err := os.Stat(fm.filePath)
|
||||
return !os.IsNotExist(err)
|
||||
}
|
||||
|
||||
func (fm *Manifest) Read() (uint32, uint32, error) {
|
||||
content, err := os.ReadFile(fm.filePath)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if len(content) == 0 {
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
parts := strings.Split(string(content), "|")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("file content is not in expected format")
|
||||
}
|
||||
|
||||
part1, err := strconv.ParseUint(strings.TrimSpace(parts[0]), 10, 32)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error parsing the first part: %v", err)
|
||||
}
|
||||
|
||||
part2, err := strconv.ParseUint(strings.TrimSpace(parts[1]), 10, 32)
|
||||
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error parsing the second part: %v", err)
|
||||
}
|
||||
|
||||
return uint32(part1), uint32(part2), nil
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
package ledgers
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestManifest(t *testing.T) {
|
||||
manifest := NewManifest("testdata")
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
assert.False(t, manifest.IsExist())
|
||||
_, _, err := manifest.Read()
|
||||
assert.Error(t, err)
|
||||
|
||||
err = manifest.SetLedgerRange(1, 10)
|
||||
assert.NoError(t, err)
|
||||
err = manifest.AppendDeltaLedger(11, 20)
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, manifest.IsExist())
|
||||
err = manifest.AppendDeltaLedger(22, 30)
|
||||
assert.Error(t, err)
|
||||
|
||||
start, end, err := manifest.Read()
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, start, uint32(1))
|
||||
assert.Equal(t, end, uint32(20))
|
||||
}
|
||||
|
||||
func TestManifestInvalidPath(t *testing.T) {
|
||||
manifest := NewManifest("/")
|
||||
|
||||
assert.False(t, manifest.IsExist())
|
||||
_, _, err := manifest.Read()
|
||||
assert.Error(t, err)
|
||||
|
||||
err = manifest.SetLedgerRange(1, 10)
|
||||
assert.Error(t, err)
|
||||
}
|
||||
@@ -13,9 +13,9 @@ type Args struct {
|
||||
EndSeq uint32
|
||||
Path string
|
||||
GrpcServer string
|
||||
WsServer string
|
||||
ServerMode bool
|
||||
ShowRange bool
|
||||
GrpcPort uint32
|
||||
WsPort uint32
|
||||
}
|
||||
|
||||
func Parse() (*Args, error) {
|
||||
@@ -27,10 +27,10 @@ func Parse() (*Args, error) {
|
||||
seq := fs.Uint32("start_seq", 0, "Starting sequence number")
|
||||
endSeq := fs.Uint32("end_seq", 0, "Ending sequence number")
|
||||
path := fs.StringP("path", "p", "", "Path to the data")
|
||||
grpcServer := fs.StringP("grpc_server", "g", "0.0.0.0:50051", "rippled's gRPC server address")
|
||||
wsServer := fs.StringP("ws_server", "w", "0.0.0.0:6006", "rippled's gRPC server address")
|
||||
grpcServer := fs.StringP("grpc_server", "g", "localhost:50051", "rippled's gRPC server address")
|
||||
serverMode := fs.BoolP("server", "s", false, "Start server mode")
|
||||
showRange := fs.BoolP("range", "r", false, "Show the range of the snapshot")
|
||||
grpcPort := fs.Uint32("grpc_port", 0, "Port for gRPC server to listen on")
|
||||
wsPort := fs.Uint32("ws_port", 0, "Port for WebSocket server to listen on")
|
||||
fs.Parse(os.Args[1:])
|
||||
|
||||
if *serverMode && *exportMode != "" {
|
||||
@@ -38,26 +38,22 @@ func Parse() (*Args, error) {
|
||||
}
|
||||
|
||||
if *serverMode {
|
||||
if *grpcServer == "" || *wsServer == "" || *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --grpc_server and --ws_server and --path are required for server mode.")
|
||||
if *grpcPort == 0 || *wsPort == 0 || *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --grpc_port and --ws_port and --path are required for server mode.")
|
||||
}
|
||||
} else if *exportMode != "" {
|
||||
if *exportMode == "full" || *exportMode == "delta" {
|
||||
if *seq == 0 || *endSeq == 0 || *path == "" || *grpcServer == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.")
|
||||
return nil, fmt.Errorf("Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: Invalid export mode. Use 'full' or 'delta'.")
|
||||
}
|
||||
} else if *showRange {
|
||||
if *path == "" {
|
||||
return nil, fmt.Errorf("Invalid usage: --path is required for show range.")
|
||||
}
|
||||
} else {
|
||||
return nil, fmt.Errorf("Invalid usage: --export or --server or --range flag is required.")
|
||||
return nil, fmt.Errorf("Invalid usage: --export or --server flag is required.")
|
||||
}
|
||||
|
||||
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *wsServer, *serverMode, *showRange}, nil
|
||||
return &Args{*exportMode, *seq, *endSeq, *path, *grpcServer, *serverMode, *grpcPort, *wsPort}, nil
|
||||
}
|
||||
|
||||
func PrintUsage() {
|
||||
|
||||
@@ -25,7 +25,6 @@ func TestParse(t *testing.T) {
|
||||
EndSeq: 10,
|
||||
Path: "/data",
|
||||
GrpcServer: "localhost:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
ServerMode: false,
|
||||
},
|
||||
expectErr: false,
|
||||
@@ -35,7 +34,7 @@ func TestParse(t *testing.T) {
|
||||
args: []string{"cmd", "--export=delta", "--start_seq=1"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export.",
|
||||
errMessage: "Invalid usage: --start_seq, --end_seq, --grpc_server and --path are required for export",
|
||||
},
|
||||
{
|
||||
name: "Invalid export mode",
|
||||
@@ -46,24 +45,18 @@ func TestParse(t *testing.T) {
|
||||
},
|
||||
{
|
||||
name: "Server mode with default grpc server flags",
|
||||
args: []string{"cmd", "--server", "--path=/server_data"},
|
||||
args: []string{"cmd", "--server", "--ws_port=1234", "--grpc_port=22", "--path=/server_data"},
|
||||
want: &Args{
|
||||
ServerMode: true,
|
||||
GrpcPort: 22,
|
||||
WsPort: 1234,
|
||||
StartSeq: 0,
|
||||
EndSeq: 0,
|
||||
Path: "/server_data",
|
||||
GrpcServer: "0.0.0.0:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
GrpcServer: "localhost:50051",
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
{
|
||||
name: "Server mode with empty grpc server flag",
|
||||
args: []string{"cmd", "--server", "--grpc_server=", "--path=/server_data"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --grpc_server and --ws_server and --path are required for server mode.",
|
||||
},
|
||||
{
|
||||
name: "Server and export mode together (error)",
|
||||
args: []string{"cmd", "--server", "--export=full"},
|
||||
@@ -71,24 +64,6 @@ func TestParse(t *testing.T) {
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --server and --export cannot be used at the same time.",
|
||||
},
|
||||
{
|
||||
name: "Show range without path",
|
||||
args: []string{"cmd", "--range"},
|
||||
want: nil,
|
||||
expectErr: true,
|
||||
errMessage: "Invalid usage: --path is required for show range.",
|
||||
},
|
||||
{
|
||||
name: "Show range",
|
||||
args: []string{"cmd", "--range", "--path=/range_data"},
|
||||
want: &Args{
|
||||
ShowRange: true,
|
||||
Path: "/range_data",
|
||||
GrpcServer: "0.0.0.0:50051",
|
||||
WsServer: "0.0.0.0:6006",
|
||||
},
|
||||
expectErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
// create a server implement the xrpl rpc v1 server interface
|
||||
type Server struct {
|
||||
pb.XRPLedgerAPIServiceServer
|
||||
ledgersHouse *ledgers.LedgersHouse
|
||||
}
|
||||
|
||||
func (s *Server) GetLedger(ctx context.Context, req *pb.GetLedgerRequest) (*pb.GetLedgerResponse, error) {
|
||||
return s.ledgersHouse.ReadLedgerDeltaData(req.GetLedger().GetSequence())
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerData(ctx context.Context, req *pb.GetLedgerDataRequest) (*pb.GetLedgerDataResponse, error) {
|
||||
marker := req.GetMarker()
|
||||
if marker == nil {
|
||||
marker = make([]byte, 32)
|
||||
}
|
||||
return s.ledgersHouse.ReadLedgerData(req.GetLedger().GetSequence(), marker)
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerDiff(ctx context.Context, req *pb.GetLedgerDiffRequest) (*pb.GetLedgerDiffResponse, error) {
|
||||
return nil, fmt.Errorf("GetLedgerDiff not supported")
|
||||
}
|
||||
|
||||
func (s *Server) GetLedgerEntry(ctx context.Context, req *pb.GetLedgerEntryRequest) (*pb.GetLedgerEntryResponse, error) {
|
||||
return nil, fmt.Errorf("GetLedgerEntry not supported")
|
||||
}
|
||||
|
||||
func newServer(path string) *Server {
|
||||
s := &Server{}
|
||||
s.ledgersHouse = ledgers.NewLedgersHouse(path)
|
||||
return s
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
)
|
||||
|
||||
func TestUnavaibleMethods(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
|
||||
req := &pb.GetLedgerDiffRequest{}
|
||||
_, err := srv.GetLedgerDiff(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "GetLedgerDiff not supported")
|
||||
|
||||
req2 := &pb.GetLedgerEntryRequest{}
|
||||
_, err = srv.GetLedgerEntry(context.Background(), req2)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "GetLedgerEntry not supported")
|
||||
}
|
||||
|
||||
func TestWhenPathIsInvalid(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
|
||||
req := &pb.GetLedgerRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := srv.GetLedger(context.Background(), req)
|
||||
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "open testdata/ledger_diff_0/2.dat: no such file or directory")
|
||||
|
||||
req2 := &pb.GetLedgerDataRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 2,
|
||||
},
|
||||
},
|
||||
}
|
||||
_, err = srv.GetLedgerData(context.Background(), req2)
|
||||
assert.Error(t, err)
|
||||
assert.Equal(t, err.Error(), "open testdata/ledger_data_2/marker_0000000000000000000000000000000000000000000000000000000000000000/0000000000000000000000000000000000000000000000000000000000000000.dat: no such file or directory")
|
||||
}
|
||||
|
||||
func TestWhenPathIsValid(t *testing.T) {
|
||||
srv := newServer("testdata")
|
||||
ledger := ledgers.NewLedgersHouse("testdata")
|
||||
defer os.RemoveAll("testdata")
|
||||
|
||||
marker := [32]byte{}
|
||||
ledger.WriteLedgerData(1, marker[:], &pb.GetLedgerDataResponse{})
|
||||
ledger.WriteLedgerDeltaData(1, &pb.GetLedgerResponse{})
|
||||
|
||||
req := &pb.GetLedgerRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
res, err := srv.GetLedger(context.Background(), req)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res)
|
||||
|
||||
req2 := &pb.GetLedgerDataRequest{
|
||||
Ledger: &pb.LedgerSpecifier{
|
||||
Ledger: &pb.LedgerSpecifier_Sequence{
|
||||
Sequence: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
res2, err := srv.GetLedgerData(context.Background(), req2)
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, res2)
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
pb "xrplf/clio/clio_snapshot/org/xrpl/rpc/v1"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
)
|
||||
|
||||
func StartServer(grpcServerAddr string, wsServerAddr string, path string) {
|
||||
ledgersHouse := ledgers.NewLedgersHouse(path)
|
||||
|
||||
if !ledgersHouse.IsExist() {
|
||||
log.Fatalf("Can't start server againist invalid snapshot folder: %s", path)
|
||||
}
|
||||
|
||||
startSeq, endSeq, err := ledgersHouse.GetRange()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get range: %v", err)
|
||||
}
|
||||
|
||||
lis, err := net.Listen("tcp", grpcServerAddr)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to listen: %v", err)
|
||||
}
|
||||
|
||||
grpcServer := grpc.NewServer()
|
||||
pb.RegisterXRPLedgerAPIServiceServer(grpcServer, newServer(path))
|
||||
log.Print("Starting server...")
|
||||
go grpcServer.Serve(lis)
|
||||
|
||||
wsServer := NewWebSocketServer("Snapshot Server", func(message string) string {
|
||||
//mimic the response of the ledger stream
|
||||
ledgerStreamReply := fmt.Sprintf("{\"fee_base\":10,\"ledger_hash\":\"A320C67DA7D1250A577AC5AACDF06ADC25E0EEEF7AE5B8D63CE2E1CC7F76A438\",\"ledger_index\":%d,\"ledger_time\":792853443,\"reserve_base\":1000000,\"reserve_inc\":200000,\"txn_count\":0,\"type\":\"ledgerClosed\",\"validated_ledgers\":\"%d-%d\"}",
|
||||
endSeq, startSeq, endSeq)
|
||||
return ledgerStreamReply
|
||||
})
|
||||
wsServer.Start(wsServerAddr)
|
||||
|
||||
select {}
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type WebSocketServer struct {
|
||||
serverName string
|
||||
callback func(message string) string
|
||||
upgrader websocket.Upgrader
|
||||
}
|
||||
|
||||
func NewWebSocketServer(serverName string, callback func(message string) string) *WebSocketServer {
|
||||
return &WebSocketServer{
|
||||
serverName: serverName,
|
||||
callback: callback,
|
||||
upgrader: websocket.Upgrader{
|
||||
CheckOrigin: func(r *http.Request) bool { return true }, // Allow all connections
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (ws *WebSocketServer) handleConnections() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
conn, err := ws.upgrader.Upgrade(w, r, nil)
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error upgrading to WebSocket: %v", ws.serverName, err)
|
||||
return
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
log.Printf("[%s] New WebSocket connection established", ws.serverName)
|
||||
|
||||
for {
|
||||
_, msg, err := conn.ReadMessage()
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error reading message: %v", ws.serverName, err)
|
||||
break
|
||||
}
|
||||
log.Printf("[%s] Received: %s", ws.serverName, msg)
|
||||
|
||||
response := ws.callback(string(msg))
|
||||
|
||||
err = conn.WriteMessage(websocket.TextMessage, []byte(response))
|
||||
log.Printf("[%s] Sending: %s", ws.serverName, response)
|
||||
if err != nil {
|
||||
log.Printf("[%s] Error writing message: %v", ws.serverName, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ws *WebSocketServer) Start(address string) {
|
||||
http.HandleFunc("/", ws.handleConnections())
|
||||
log.Printf("[%s] Starting ws server on address: %s", ws.serverName, address)
|
||||
err := http.ListenAndServe(address, nil)
|
||||
if err != nil {
|
||||
log.Fatalf("[%s] Server failed: %v", ws.serverName, err)
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,7 @@ import (
|
||||
"log"
|
||||
|
||||
"xrplf/clio/clio_snapshot/internal/export"
|
||||
"xrplf/clio/clio_snapshot/internal/ledgers"
|
||||
"xrplf/clio/clio_snapshot/internal/parse_args"
|
||||
"xrplf/clio/clio_snapshot/internal/server"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -20,18 +18,6 @@ func main() {
|
||||
|
||||
} else if args.ExportMode == "delta" {
|
||||
export.ExportFromDeltaLedger(args.GrpcServer, args.StartSeq, args.EndSeq, args.Path)
|
||||
} else if args.ServerMode {
|
||||
server.StartServer(args.GrpcServer, args.WsServer, args.Path)
|
||||
} else if args.ShowRange {
|
||||
ledgers := ledgers.NewLedgersHouse(args.Path)
|
||||
if !ledgers.IsExist() {
|
||||
log.Fatalf("Invalid snapshot folder: %s", args.Path)
|
||||
}
|
||||
startSeq, endSeq, err := ledgers.GetRange()
|
||||
if err == nil {
|
||||
log.Printf("Snapshot range: %d-%d", startSeq, endSeq)
|
||||
} else {
|
||||
log.Fatalf("Failed to get snapshot range: %v", err)
|
||||
}
|
||||
}
|
||||
//TODO: Implement server mode
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user