ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-25 06:05:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

partial merge needs fixing

Browse Source
This commit is contained in:
CJ Cobb
2021-06-07 21:35:33 +00:00
parent 6f6c573644 5f6edee312
commit 51ff08b3ae
34 changed files with 2857 additions and 1542 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
CMakeLists.txt
View File

@@ -10,7 +10,7 @@
set(CMAKE_VERBOSE_MAKEFILE TRUE)
project(reporting)
cmake_minimum_required(VERSION 3.16)
set (CMAKE_CXX_STANDARD 17)
set (CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -Wno-narrowing")
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_MULTITHREADED ON)
@@ -21,10 +21,22 @@ endif ()
file (TO_CMAKE_PATH "${BOOST_ROOT}" BOOST_ROOT)
FIND_PACKAGE( Boost 1.75 COMPONENTS filesystem log log_setup thread system REQUIRED )
add_executable (reporting
websocket_server_async.cpp
include(FetchContent)
FetchContent_Declare(
googletest
URL https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip
)
FetchContent_MakeAvailable(googletest)
enable_testing()
include(GoogleTest)
add_executable (reporting_main
server/websocket_server_async.cpp
)
add_executable (reporting_tests
unittests/main.cpp
)
add_library(reporting reporting/BackendInterface.h)
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps")
include(ExternalProject)
message(${CMAKE_CURRENT_BINARY_DIR})
@@ -59,6 +71,7 @@ target_sources(reporting PRIVATE
reporting/CassandraBackend.cpp
reporting/PostgresBackend.cpp
reporting/BackendIndexer.cpp
reporting/BackendInterface.cpp
reporting/Pg.cpp
reporting/P2pProxy.cpp
reporting/DBHelpers.cpp
@@ -74,6 +87,7 @@ target_sources(reporting PRIVATE
handlers/LedgerRange.cpp
handlers/Ledger.cpp
handlers/LedgerEntry.cpp
<<<<<<< HEAD
handlers/AccountChannels.cpp
handlers/AccountLines.cpp
handlers/AccountCurrencies.cpp
@@ -82,8 +96,17 @@ target_sources(reporting PRIVATE
handlers/ChannelAuthorize.cpp
handlers/ChannelVerify.cpp
handlers/Subscribe.cpp)
=======
handlers/ServerInfo.cpp)
>>>>>>> dev
message(${Boost_LIBRARIES})
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR} ${Boost_INCLUDE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
TARGET_LINK_LIBRARIES(reporting PUBLIC ${Boost_LIBRARIES})
TARGET_LINK_LIBRARIES(reporting_main PUBLIC reporting)
TARGET_LINK_LIBRARIES(reporting_tests PUBLIC reporting gtest_main)
gtest_discover_tests(reporting_tests)

4
deps/cassandra.cmake vendored
View File

@@ -97,7 +97,6 @@ if(NOT cassandra)
file(TO_CMAKE_PATH "${libuv_src_SOURCE_DIR}" libuv_src_SOURCE_DIR)
endif()
add_library (cassandra STATIC IMPORTED GLOBAL)
ExternalProject_Add(cassandra_src
PREFIX ${nih_cache_path}
@@ -148,6 +147,9 @@ if(NOT cassandra)
else()
target_link_libraries(cassandra INTERFACE ${zlib})
endif()
set(OPENSSL_USE_STATIC_LIBS TRUE)
find_package(OpenSSL REQUIRED)
target_link_libraries(cassandra INTERFACE OpenSSL::SSL)
file(TO_CMAKE_PATH "${cassandra_src_SOURCE_DIR}" cassandra_src_SOURCE_DIR)
target_link_libraries(reporting PUBLIC cassandra)

4
handlers/AccountInfo.cpp
View File

@@ -104,7 +104,7 @@ doAccountInfo(
{
response["success"] = "fetched successfully!";
if (!binary)
response["object"] = getJson(sle);
response["object"] = toJson(sle);
else
response["object"] = ripple::strHex(*dbResponse);
response["db_time"] = time;
@@ -124,7 +124,7 @@ doAccountInfo(
// support multiple SignerLists on one account.
auto const sleSigners = ledger->read(keylet::signers(accountID));
if (sleSigners)
jvSignerList.append(sleSigners->getJson(JsonOptions::none));
jvSignerList.append(sleSigners->toJson(JsonOptions::none));
result[jss::account_data][jss::signer_lists] =
std::move(jvSignerList);

107
handlers/AccountTx.cpp
View File

@@ -19,102 +19,6 @@
#include <handlers/RPCHelpers.h>
#include <reporting/BackendInterface.h>
#include <reporting/Pg.h>
std::vector<std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>>
doAccountTxStoredProcedure(
ripple::AccountID const& account,
std::shared_ptr<PgPool>& pgPool,
BackendInterface const& backend)
{
pg_params dbParams;
char const*& command = dbParams.first;
std::vector<std::optional<std::string>>& values = dbParams.second;
command =
"SELECT account_tx($1::bytea, $2::bool, "
"$3::bigint, $4::bigint, $5::bigint, $6::bytea, "
"$7::bigint, $8::bool, $9::bigint, $10::bigint)";
values.resize(10);
values[0] = "\\x" + ripple::strHex(account);
values[1] = "true";
static std::uint32_t const page_length(200);
values[2] = std::to_string(page_length);
auto res = PgQuery(pgPool)(dbParams);
if (!res)
{
BOOST_LOG_TRIVIAL(error)
<< __func__ << " : Postgres response is null - account = "
<< ripple::strHex(account);
assert(false);
return {};
}
else if (res.status() != PGRES_TUPLES_OK)
{
assert(false);
return {};
}
if (res.isNull() || res.ntuples() == 0)
{
BOOST_LOG_TRIVIAL(error)
<< __func__ << " : No data returned from Postgres : account = "
<< ripple::strHex(account);
assert(false);
return {};
}
char const* resultStr = res.c_str();
boost::json::object result = boost::json::parse(resultStr).as_object();
if (result.contains("transactions"))
{
std::vector<ripple::uint256> nodestoreHashes;
for (auto& t : result.at("transactions").as_array())
{
boost::json::object obj = t.as_object();
if (obj.contains("ledger_seq") && obj.contains("nodestore_hash"))
{
std::string nodestoreHashHex =
obj.at("nodestore_hash").as_string().c_str();
nodestoreHashHex.erase(0, 2);
ripple::uint256 nodestoreHash;
if (!nodestoreHash.parseHex(nodestoreHashHex))
assert(false);
if (nodestoreHash.isNonZero())
{
nodestoreHashes.push_back(nodestoreHash);
}
else
{
assert(false);
}
}
else
{
assert(false);
}
}
std::vector<std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>>
results;
auto dbResults = backend.fetchTransactions(nodestoreHashes);
for (auto const& res : dbResults)
{
if (res.transaction.size() && res.metadata.size())
results.push_back(deserializeTxPlusMeta(res));
}
return results;
}
return {};
}
// {
// account: account,
@@ -190,7 +94,9 @@ doAccountTx(boost::json::object const& request, BackendInterface const& backend)
auto [blobs, retCursor] =
backend.fetchAccountTransactions(*account, limit, cursor);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info) << __func__ << " db fetch took " << ((end - start).count() / 1000000000.0) << " num blobs = " << blobs.size();
BOOST_LOG_TRIVIAL(info) << __func__ << " db fetch took "
<< ((end - start).count() / 1000000000.0)
<< " num blobs = " << blobs.size();
for (auto const& txnPlusMeta : blobs)
{
if (txnPlusMeta.ledgerSequence > ledgerSequence)
@@ -204,8 +110,8 @@ doAccountTx(boost::json::object const& request, BackendInterface const& backend)
if (!binary)
{
auto [txn, meta] = deserializeTxPlusMeta(txnPlusMeta);
obj["transaction"] = getJson(*txn);
obj["metadata"] = getJson(*meta);
obj["transaction"] = toJson(*txn);
obj["metadata"] = toJson(*meta);
}
else
{
@@ -224,7 +130,8 @@ doAccountTx(boost::json::object const& request, BackendInterface const& backend)
response["cursor"] = cursorJson;
}
auto end2 = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info) << __func__ << " serialization took " << ((end2 - end).count() / 1000000000.0);
BOOST_LOG_TRIVIAL(info) << __func__ << " serialization took "
<< ((end2 - end).count() / 1000000000.0);
return response;
}

2
handlers/BookOffers.cpp
View File

@@ -272,7 +272,7 @@ doBookOffers(
ripple::SLE offer{it, obj.key};
ripple::uint256 bookDir = offer.getFieldH256(ripple::sfBookDirectory);
boost::json::object offerJson = getJson(offer);
boost::json::object offerJson = toJson(offer);
offerJson["quality"] = ripple::amountFromQuality(getQuality(bookDir)).getText();
jsonOffers.push_back(offerJson);
}

34
handlers/Ledger.cpp
View File

@@ -1,21 +1,5 @@
#include <handlers/RPCHelpers.h>
#include <reporting/BackendInterface.h>
std::vector<unsigned char>
ledgerInfoToBlob(ripple::LedgerInfo const& info)
{
ripple::Serializer s;
s.add32(info.seq);
s.add64(info.drops.drops());
s.addBitString(info.parentHash);
s.addBitString(info.txHash);
s.addBitString(info.accountHash);
s.add32(info.parentCloseTime.time_since_epoch().count());
s.add32(info.closeTime.time_since_epoch().count());
s.add8(info.closeTimeResolution.count());
s.add8(info.closeFlags);
// s.addBitString(info.hash);
return s.peekData();
}
boost::json::object
doLedger(boost::json::object const& request, BackendInterface const& backend)
@@ -53,19 +37,7 @@ doLedger(boost::json::object const& request, BackendInterface const& backend)
}
else
{
header["ledger_sequence"] = lgrInfo->seq;
header["ledger_hash"] = ripple::strHex(lgrInfo->hash);
header["txns_hash"] = ripple::strHex(lgrInfo->txHash);
header["state_hash"] = ripple::strHex(lgrInfo->accountHash);
header["parent_hash"] = ripple::strHex(lgrInfo->parentHash);
header["total_coins"] = ripple::to_string(lgrInfo->drops);
header["close_flags"] = lgrInfo->closeFlags;
// Always show fields that contribute to the ledger hash
header["parent_close_time"] =
lgrInfo->parentCloseTime.time_since_epoch().count();
header["close_time"] = lgrInfo->closeTime.time_since_epoch().count();
header["close_time_resolution"] = lgrInfo->closeTimeResolution.count();
header = toJson(*lgrInfo);
}
response["header"] = header;
if (getTransactions)
@@ -86,8 +58,8 @@ doLedger(boost::json::object const& request, BackendInterface const& backend)
if (!binary)
{
auto [sttx, meta] = deserializeTxPlusMeta(obj);
entry["transaction"] = getJson(*sttx);
entry["metadata"] = getJson(*meta);
entry["transaction"] = toJson(*sttx);
entry["metadata"] = toJson(*meta);
}
else
{

9
handlers/LedgerData.cpp
View File

@@ -59,7 +59,12 @@ doLedgerData(
{
BOOST_LOG_TRIVIAL(debug) << __func__ << " : parsing cursor";
cursor = ripple::uint256{};
cursor->parseHex(request.at("cursor").as_string().c_str());
if (!cursor->parseHex(request.at("cursor").as_string().c_str()))
{
response["error"] = "Invalid cursor";
response["request"] = request;
return response;
}
}
bool binary =
request.contains("binary") ? request.at("binary").as_bool() : false;
@@ -91,7 +96,7 @@ doLedgerData(
objects.push_back(entry);
}
else
objects.push_back(getJson(sle));
objects.push_back(toJson(sle));
}
response["objects"] = objects;
if (returnedCursor)

2
handlers/LedgerEntry.cpp
View File

@@ -47,7 +47,7 @@ doLedgerEntry(
{
ripple::STLedgerEntry sle{
ripple::SerialIter{dbResponse->data(), dbResponse->size()}, key};
response["object"] = getJson(sle);
response["object"] = toJson(sle);
}
return response;

45
handlers/RPCHelpers.cpp
View File

@@ -69,7 +69,7 @@ deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_
}
boost::json::object
getJson(ripple::STBase const& obj)
toJson(ripple::STBase const& obj)
{
auto start = std::chrono::system_clock::now();
boost::json::value value = boost::json::parse(
@@ -82,6 +82,7 @@ getJson(ripple::STBase const& obj)
}
boost::json::object
<<<<<<< HEAD
getJson(ripple::TxMeta const& meta)
{
auto start = std::chrono::system_clock::now();
@@ -105,6 +106,9 @@ getJson(Json::Value const& value)
boost::json::object
getJson(ripple::SLE const& sle)
=======
toJson(ripple::SLE const& sle)
>>>>>>> dev
{
auto start = std::chrono::system_clock::now();
boost::json::value value = boost::json::parse(
@@ -115,6 +119,27 @@ getJson(ripple::SLE const& sle)
.count();
return value.as_object();
}
boost::json::object
toJson(ripple::LedgerInfo const& lgrInfo)
{
boost::json::object header;
header["ledger_sequence"] = lgrInfo.seq;
header["ledger_hash"] = ripple::strHex(lgrInfo.hash);
header["txns_hash"] = ripple::strHex(lgrInfo.txHash);
header["state_hash"] = ripple::strHex(lgrInfo.accountHash);
header["parent_hash"] = ripple::strHex(lgrInfo.parentHash);
header["total_coins"] = ripple::to_string(lgrInfo.drops);
header["close_flags"] = lgrInfo.closeFlags;
// Always show fields that contribute to the ledger hash
header["parent_close_time"] =
lgrInfo.parentCloseTime.time_since_epoch().count();
header["close_time"] = lgrInfo.closeTime.time_since_epoch().count();
header["close_time_resolution"] = lgrInfo.closeTimeResolution.count();
return header;
}
std::optional<uint32_t>
ledgerSequenceFromRequest(
boost::json::object const& request,
@@ -129,6 +154,7 @@ ledgerSequenceFromRequest(
return request.at("ledger_index").as_int64();
}
}
<<<<<<< HEAD
std::optional<ripple::uint256>
traverseOwnedNodes(
@@ -380,4 +406,21 @@ getAccountsFromTransaction(boost::json::object const& transaction)
}
return accounts;
=======
std::vector<unsigned char>
ledgerInfoToBlob(ripple::LedgerInfo const& info)
{
ripple::Serializer s;
s.add32(info.seq);
s.add64(info.drops.drops());
s.addBitString(info.parentHash);
s.addBitString(info.txHash);
s.addBitString(info.accountHash);
s.add32(info.parentCloseTime.time_since_epoch().count());
s.add32(info.closeTime.time_since_epoch().count());
s.add8(info.closeTimeResolution.count());
s.add8(info.closeFlags);
s.addBitString(info.hash);
return s.peekData();
>>>>>>> dev
}

12
handlers/RPCHelpers.h
View File

@@ -21,10 +21,13 @@ std::pair<
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs, std::uint32_t seq);
boost::json::object
getJson(ripple::STBase const& obj);
toJson(ripple::STBase const& obj);
boost::json::object
getJson(ripple::SLE const& sle);
toJson(ripple::SLE const& sle);
boost::json::object
toJson(ripple::LedgerInfo const& info);
boost::json::object
getJson(ripple::TxMeta const& meta);
@@ -37,6 +40,7 @@ ledgerSequenceFromRequest(
boost::json::object const& request,
BackendInterface const& backend);
<<<<<<< HEAD
std::optional<ripple::uint256>
traverseOwnedNodes(
BackendInterface const& backend,
@@ -52,5 +56,9 @@ keypairFromRequst(
std::vector<ripple::AccountID>
getAccountsFromTransaction(boost::json::object const& transaction);
=======
std::vector<unsigned char>
ledgerInfoToBlob(ripple::LedgerInfo const& info);
>>>>>>> dev
#endif

53
handlers/ServerInfo.cpp Normal file
View File

@@ -0,0 +1,53 @@
#include <handlers/RPCHelpers.h>
#include <reporting/BackendInterface.h>
boost::json::object
doServerInfo(
boost::json::object const& request,
BackendInterface const& backend)
{
boost::json::object response;
auto rng = backend.fetchLedgerRange();
if (!rng)
{
response["complete_ledgers"] = "empty";
}
else
{
std::string completeLedgers = std::to_string(rng->minSequence);
if (rng->maxSequence != rng->minSequence)
completeLedgers += "-" + std::to_string(rng->maxSequence);
response["complete_ledgers"] = completeLedgers;
}
if (rng)
{
auto lgrInfo = backend.fetchLedgerBySequence(rng->maxSequence);
response["validated_ledger"] = toJson(*lgrInfo);
}
boost::json::array indexes;
if (rng)
{
uint32_t cur = rng->minSequence;
while (cur <= rng->maxSequence + 1)
{
auto keyIndex = backend.getKeyIndexOfSeq(cur);
assert(keyIndex.has_value());
cur = keyIndex->keyIndex;
boost::json::object entry;
entry["complete"] = backend.isLedgerIndexed(cur);
entry["sequence"] = cur;
indexes.emplace_back(entry);
cur = cur + 1;
}
}
response["indexes"] = indexes;
auto indexing = backend.getIndexer().getCurrentlyIndexing();
if (indexing)
response["indexing"] = *indexing;
else
response["indexing"] = "none";
return response;
}

4
handlers/Tx.cpp
View File

@@ -63,8 +63,8 @@ doTx(boost::json::object const& request, BackendInterface const& backend)
if (!binary)
{
auto [sttx, meta] = deserializeTxPlusMeta(dbResponse.value());
response["transaction"] = getJson(*sttx);
response["metadata"] = getJson(*meta);
response["transaction"] = toJson(*sttx);
response["metadata"] = toJson(*meta);
}
else
{

11
reporting/BackendFactory.h
View File

@@ -10,9 +10,13 @@ namespace Backend {
std::unique_ptr<BackendInterface>
make_Backend(boost::json::object const& config)
{
<<<<<<< HEAD
BOOST_LOG_TRIVIAL(info) << __func__ << ": Constructing BackendInterface";
boost::json::object const& dbConfig = config.at("database").as_object();
=======
boost::json::object dbConfig = config.at("database").as_object();
>>>>>>> dev
bool readOnly = false;
if (config.contains("read_only"))
@@ -24,7 +28,14 @@ make_Backend(boost::json::object const& config)
if (boost::iequals(type, "cassandra"))
{
<<<<<<< HEAD
backend =
=======
if (config.contains("online_delete"))
dbConfig.at(type).as_object()["ttl"] =
config.at("online_delete").as_int64() * 4;
auto backend =
>>>>>>> dev
std::make_unique<CassandraBackend>(dbConfig.at(type).as_object());
}
else if (boost::iequals(type, "postgres"))

483
reporting/BackendIndexer.cpp
View File

@@ -2,236 +2,25 @@
namespace Backend {
BackendIndexer::BackendIndexer(boost::json::object const& config)
: strand_(ioc_)
{
if (config.contains("indexer_key_shift"))
keyShift_ = config.at("indexer_key_shift").as_int64();
if (config.contains("indexer_book_shift"))
bookShift_ = config.at("indexer_book_shift").as_int64();
work_.emplace(ioc_);
ioThread_ = std::thread{[this]() { ioc_.run(); }};
};
BackendIndexer::~BackendIndexer()
{
std::unique_lock lck(mutex_);
work_.reset();
ioThread_.join();
}
void
BackendIndexer::addKey(ripple::uint256 const& key)
BackendIndexer::addKey(ripple::uint256&& key)
{
std::unique_lock lck(mtx);
keys.insert(key);
keysCumulative.insert(key);
}
void
BackendIndexer::addKeyAsync(ripple::uint256 const& key)
{
std::unique_lock lck(mtx);
keysCumulative.insert(key);
}
void
BackendIndexer::deleteKey(ripple::uint256 const& key)
{
std::unique_lock lck(mtx);
keysCumulative.erase(key);
if (populatingCacheAsync)
deletedKeys.insert(key);
keys.insert(std::move(key));
}
void
BackendIndexer::addBookOffer(
ripple::uint256 const& book,
ripple::uint256 const& offerKey)
{
std::unique_lock lck(mtx);
books[book].insert(offerKey);
booksCumulative[book].insert(offerKey);
}
void
BackendIndexer::addBookOfferAsync(
ripple::uint256 const& book,
ripple::uint256 const& offerKey)
{
std::unique_lock lck(mtx);
booksCumulative[book].insert(offerKey);
}
void
BackendIndexer::deleteBookOffer(
ripple::uint256 const& book,
ripple::uint256 const& offerKey)
{
std::unique_lock lck(mtx);
booksCumulative[book].erase(offerKey);
if (populatingCacheAsync)
deletedBooks[book].insert(offerKey);
}
void
writeKeyFlagLedger(
uint32_t ledgerSequence,
uint32_t shift,
BackendInterface const& backend,
std::unordered_set<ripple::uint256> const& keys)
{
uint32_t nextFlag = ((ledgerSequence >> shift << shift) + (1 << shift));
ripple::uint256 zero = {};
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " keys.size() = " << std::to_string(keys.size());
while (true)
{
try
{
auto [objects, curCursor, warning] =
backend.fetchLedgerPage({}, nextFlag, 1);
if (!warning)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " flag ledger already written. sequence = "
<< std::to_string(ledgerSequence)
<< " next flag = " << std::to_string(nextFlag)
<< "returning";
return;
}
break;
}
catch (DatabaseTimeout& t)
{
;
}
}
auto start = std::chrono::system_clock::now();
backend.writeKeys(keys, nextFlag, true);
backend.writeKeys({zero}, nextFlag, true);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " keys.size() = " << std::to_string(keys.size())
<< std::chrono::duration_cast<std::chrono::seconds>(end - start)
.count();
}
void
writeBookFlagLedger(
uint32_t ledgerSequence,
uint32_t shift,
BackendInterface const& backend,
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books)
{
uint32_t nextFlag = ((ledgerSequence >> shift << shift) + (1 << shift));
ripple::uint256 zero = {};
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " books.size() = " << std::to_string(books.size());
auto start = std::chrono::system_clock::now();
backend.writeBooks(books, nextFlag, true);
backend.writeBooks({{zero, {zero}}}, nextFlag, true);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " books.size() = " << std::to_string(books.size()) << " time = "
<< std::chrono::duration_cast<std::chrono::seconds>(end - start)
.count();
}
void
BackendIndexer::clearCaches()
{
keysCumulative = {};
booksCumulative = {};
}
void
BackendIndexer::doBooksRepair(
BackendInterface const& backend,
std::optional<uint32_t> sequence)
{
auto rng = backend.fetchLedgerRangeNoThrow();
if (!rng)
return;
if (!sequence)
sequence = rng->maxSequence;
if(sequence < rng->minSequence)
sequence = rng->minSequence;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " sequence = " << std::to_string(*sequence);
ripple::uint256 zero = {};
while (true)
{
try
{
auto [objects, cursor, warning] =
backend.fetchBookOffers(zero, *sequence, 1);
if (!warning)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " flag ledger already written. sequence = "
<< std::to_string(*sequence) << "returning";
return;
}
else
{
uint32_t lower = (*sequence - 1) >> bookShift_ << bookShift_;
doBooksRepair(backend, lower);
}
break;
}
catch (DatabaseTimeout& t)
{
;
}
}
std::optional<ripple::uint256> cursor;
while (true)
{
try
{
auto [objects, curCursor, warning] =
backend.fetchLedgerPage(cursor, *sequence, 2048);
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
cursor = curCursor;
for (auto& obj : objects)
{
if (isOffer(obj.blob))
{
auto book = getBook(obj.blob);
booksRepair[book].insert(obj.key);
}
}
if (!cursor)
break;
}
catch (DatabaseTimeout const& e)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " Database timeout fetching keys";
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
writeBookFlagLedger(*sequence, bookShift_, backend, booksRepair);
booksRepair = {};
BOOST_LOG_TRIVIAL(info)
<< __func__ << " finished. sequence = " << std::to_string(*sequence);
}
void
BackendIndexer::doKeysRepair(
BackendInterface const& backend,
@@ -245,7 +34,7 @@ BackendIndexer::doKeysRepair(
if (!sequence)
sequence = rng->maxSequence;
if(sequence < rng->minSequence)
if (sequence < rng->minSequence)
sequence = rng->minSequence;
BOOST_LOG_TRIVIAL(info)
@@ -256,35 +45,28 @@ BackendIndexer::doKeysRepair(
{
try
{
auto [objects, curCursor, warning] =
backend.fetchLedgerPage(cursor, *sequence, 2048);
// no cursor means this is the first page
if (!cursor)
if (backend.isLedgerIndexed(*sequence))
{
// if there is no warning, we don't need to do a repair
// warning only shows up on the first page
if (!warning)
{
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " flag ledger already written. returning";
return;
}
else
{
uint32_t lower = (*sequence - 1) >> keyShift_ << keyShift_;
doKeysRepair(backend, lower);
}
BOOST_LOG_TRIVIAL(info)
<< __func__ << " - " << std::to_string(*sequence)
<< " flag ledger already written. returning";
return;
}
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
cursor = curCursor;
for (auto& obj : objects)
else
{
keysRepair.insert(obj.key);
BOOST_LOG_TRIVIAL(info)
<< __func__ << " - " << std::to_string(*sequence)
<< " flag ledger not written. recursing..";
uint32_t lower = (*sequence - 1) >> keyShift_ << keyShift_;
doKeysRepair(backend, lower);
BOOST_LOG_TRIVIAL(info)
<< __func__ << " - "
<< " sequence = " << std::to_string(*sequence)
<< " lower = " << std::to_string(lower)
<< " finished recursing. submitting repair ";
writeKeyFlagLedger(lower, backend);
return;
}
if (!cursor)
break;
}
catch (DatabaseTimeout const& e)
{
@@ -293,41 +75,96 @@ BackendIndexer::doKeysRepair(
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
writeKeyFlagLedger(*sequence, keyShift_, backend, keysRepair);
keysRepair = {};
BOOST_LOG_TRIVIAL(info)
<< __func__ << " finished. sequence = " << std::to_string(*sequence);
}
void
BackendIndexer::populateCaches(BackendInterface const& backend)
BackendIndexer::doKeysRepairAsync(
BackendInterface const& backend,
std::optional<uint32_t> sequence)
{
auto rng = backend.fetchLedgerRangeNoThrow();
if (!rng)
return;
uint32_t sequence = rng->maxSequence;
boost::asio::post(strand_, [this, sequence, &backend]() {
doKeysRepair(backend, sequence);
});
}
void
BackendIndexer::writeKeyFlagLedger(
uint32_t ledgerSequence,
BackendInterface const& backend)
{
auto nextFlag = getKeyIndexOfSeq(ledgerSequence + 1);
uint32_t lower = ledgerSequence >> keyShift_ << keyShift_;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " sequence = " << std::to_string(sequence);
doBooksRepair(backend, sequence);
doKeysRepair(backend, sequence);
<< "writeKeyFlagLedger - "
<< "next flag = " << std::to_string(nextFlag.keyIndex)
<< "lower = " << std::to_string(lower)
<< "ledgerSequence = " << std::to_string(ledgerSequence) << " starting";
ripple::uint256 zero = {};
std::optional<ripple::uint256> cursor;
size_t numKeys = 0;
auto begin = std::chrono::system_clock::now();
while (true)
{
try
{
auto [objects, curCursor, warning] =
backend.fetchLedgerPage(cursor, sequence, 2048);
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
cursor = curCursor;
for (auto& obj : objects)
{
addKeyAsync(obj.key);
if (isOffer(obj.blob))
BOOST_LOG_TRIVIAL(info)
<< "writeKeyFlagLedger - checking for complete...";
if (backend.isLedgerIndexed(nextFlag.keyIndex))
{
auto book = getBook(obj.blob);
addBookOfferAsync(book, obj.key);
BOOST_LOG_TRIVIAL(warning)
<< "writeKeyFlagLedger - "
<< "flag ledger already written. flag = "
<< std::to_string(nextFlag.keyIndex)
<< " , ledger sequence = "
<< std::to_string(ledgerSequence);
return;
}
BOOST_LOG_TRIVIAL(info)
<< "writeKeyFlagLedger - is not complete";
}
indexing_ = nextFlag.keyIndex;
auto start = std::chrono::system_clock::now();
auto [objects, curCursor, warning] =
backend.fetchLedgerPage(cursor, lower, 2048);
auto mid = std::chrono::system_clock::now();
// no cursor means this is the first page
if (!cursor)
{
if (warning)
{
BOOST_LOG_TRIVIAL(error)
<< "writeKeyFlagLedger - "
<< " prev flag ledger not written "
<< std::to_string(nextFlag.keyIndex) << " : "
<< std::to_string(ledgerSequence);
assert(false);
throw std::runtime_error("Missing prev flag");
}
}
cursor = curCursor;
std::unordered_set<ripple::uint256> keys;
for (auto& obj : objects)
{
keys.insert(obj.key);
}
backend.writeKeys(keys, nextFlag, true);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(debug)
<< "writeKeyFlagLedger - " << std::to_string(nextFlag.keyIndex)
<< " fetched a page "
<< " cursor = "
<< (cursor.has_value() ? ripple::strHex(*cursor)
: std::string{})
<< " num keys = " << std::to_string(numKeys) << " fetch time = "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
mid - start)
.count()
<< " write time = "
<< std::chrono::duration_cast<std::chrono::milliseconds>(
end - mid)
.count();
if (!cursor)
break;
}
@@ -338,59 +175,16 @@ BackendIndexer::populateCaches(BackendInterface const& backend)
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
// Do reconcilation. Remove anything from keys or books that shouldn't
// be there
{
std::unique_lock lck(mtx);
populatingCacheAsync = false;
}
for (auto& key : deletedKeys)
{
deleteKey(key);
}
for (auto& book : deletedBooks)
{
for (auto& offer : book.second)
{
deleteBookOffer(book.first, offer);
}
}
{
std::unique_lock lck(mtx);
deletedKeys = {};
deletedBooks = {};
cv_.notify_one();
}
backend.writeKeys({zero}, nextFlag, true);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. keys.size() = " << std::to_string(keysCumulative.size());
<< "writeKeyFlagLedger - " << std::to_string(nextFlag.keyIndex)
<< " finished. "
<< " num keys = " << std::to_string(numKeys) << " total time = "
<< std::chrono::duration_cast<std::chrono::milliseconds>(end - begin)
.count();
indexing_ = 0;
}
void
BackendIndexer::populateCachesAsync(BackendInterface const& backend)
{
if (keysCumulative.size() > 0)
{
BOOST_LOG_TRIVIAL(info)
<< __func__ << " caches already populated. returning";
return;
}
{
std::unique_lock lck(mtx);
populatingCacheAsync = true;
}
BOOST_LOG_TRIVIAL(info) << __func__;
boost::asio::post(ioc_, [this, &backend]() { populateCaches(backend); });
}
void
BackendIndexer::waitForCaches()
{
std::unique_lock lck(mtx);
cv_.wait(lck, [this]() {
return !populatingCacheAsync && deletedKeys.size() == 0;
});
}
void
BackendIndexer::writeKeyFlagLedgerAsync(
uint32_t ledgerSequence,
@@ -400,28 +194,8 @@ BackendIndexer::writeKeyFlagLedgerAsync(
<< __func__
<< " starting. sequence = " << std::to_string(ledgerSequence);
waitForCaches();
auto keysCopy = keysCumulative;
boost::asio::post(ioc_, [=, this, &backend]() {
writeKeyFlagLedger(ledgerSequence, keyShift_, backend, keysCopy);
});
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. sequence = " << std::to_string(ledgerSequence);
}
void
BackendIndexer::writeBookFlagLedgerAsync(
uint32_t ledgerSequence,
BackendInterface const& backend)
{
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. sequence = " << std::to_string(ledgerSequence);
waitForCaches();
auto booksCopy = booksCumulative;
boost::asio::post(ioc_, [=, this, &backend]() {
writeBookFlagLedger(ledgerSequence, bookShift_, backend, booksCopy);
boost::asio::post(strand_, [this, ledgerSequence, &backend]() {
writeKeyFlagLedger(ledgerSequence, backend);
});
BOOST_LOG_TRIVIAL(info)
<< __func__
@@ -431,34 +205,37 @@ BackendIndexer::writeBookFlagLedgerAsync(
void
BackendIndexer::finish(uint32_t ledgerSequence, BackendInterface const& backend)
{
BOOST_LOG_TRIVIAL(info)
BOOST_LOG_TRIVIAL(debug)
<< __func__
<< " starting. sequence = " << std::to_string(ledgerSequence);
bool isFirst = false;
uint32_t keyIndex = getKeyIndexOfSeq(ledgerSequence);
uint32_t bookIndex = getBookIndexOfSeq(ledgerSequence);
auto rng = backend.fetchLedgerRangeNoThrow();
if (!rng || rng->minSequence == ledgerSequence)
auto keyIndex = getKeyIndexOfSeq(ledgerSequence);
if (isFirst_)
{
isFirst = true;
keyIndex = bookIndex = ledgerSequence;
auto rng = backend.fetchLedgerRangeNoThrow();
if (rng && rng->minSequence != ledgerSequence)
isFirst_ = false;
else
{
keyIndex = KeyIndex{ledgerSequence};
}
}
backend.writeKeys(keys, keyIndex);
backend.writeBooks(books, bookIndex);
if (isFirst)
{
ripple::uint256 zero = {};
backend.writeBooks({{zero, {zero}}}, ledgerSequence);
backend.writeKeys({zero}, ledgerSequence);
writeBookFlagLedgerAsync(ledgerSequence, backend);
writeKeyFlagLedgerAsync(ledgerSequence, backend);
backend.writeKeys(keys, keyIndex);
if (isFirst_)
{
// write completion record
ripple::uint256 zero = {};
backend.writeKeys({zero}, keyIndex);
// write next flag sychronously
keyIndex = getKeyIndexOfSeq(ledgerSequence + 1);
backend.writeKeys(keys, keyIndex);
backend.writeKeys({zero}, keyIndex);
}
isFirst_ = false;
keys = {};
books = {};
BOOST_LOG_TRIVIAL(info)
BOOST_LOG_TRIVIAL(debug)
<< __func__
<< " finished. sequence = " << std::to_string(ledgerSequence);
}
}
} // namespace Backend

335
reporting/BackendInterface.cpp Normal file
View File

@@ -0,0 +1,335 @@
#include <ripple/protocol/Indexes.h>
#include <ripple/protocol/STLedgerEntry.h>
#include <reporting/BackendInterface.h>
namespace Backend {
bool
BackendInterface::finishWrites(uint32_t ledgerSequence) const
{
indexer_.finish(ledgerSequence, *this);
auto commitRes = doFinishWrites();
if (commitRes)
{
if (isFirst_)
indexer_.doKeysRepairAsync(*this, ledgerSequence);
if (indexer_.isKeyFlagLedger(ledgerSequence))
indexer_.writeKeyFlagLedgerAsync(ledgerSequence, *this);
isFirst_ = false;
}
else
{
// if commitRes is false, we are relinquishing control of ETL. We
// reset isFirst_ to true so that way if we later regain control of
// ETL, we trigger the index repair
isFirst_ = true;
}
return commitRes;
}
bool
BackendInterface::isLedgerIndexed(std::uint32_t ledgerSequence) const
{
auto keyIndex = getKeyIndexOfSeq(ledgerSequence);
if (keyIndex)
{
auto page = doFetchLedgerPage({}, ledgerSequence, 1);
return !page.warning.has_value();
}
return false;
}
void
BackendInterface::writeLedgerObject(
std::string&& key,
uint32_t seq,
std::string&& blob,
bool isCreated,
bool isDeleted,
std::optional<ripple::uint256>&& book) const
{
ripple::uint256 key256 = ripple::uint256::fromVoid(key.data());
indexer_.addKey(std::move(key256));
doWriteLedgerObject(
std::move(key),
seq,
std::move(blob),
isCreated,
isDeleted,
std::move(book));
}
std::optional<LedgerRange>
BackendInterface::fetchLedgerRangeNoThrow() const
{
BOOST_LOG_TRIVIAL(warning) << __func__;
while (true)
{
try
{
return fetchLedgerRange();
}
catch (DatabaseTimeout& t)
{
;
}
}
}
std::optional<KeyIndex>
BackendInterface::getKeyIndexOfSeq(uint32_t seq) const
{
if (indexer_.isKeyFlagLedger(seq))
return KeyIndex{seq};
auto rng = fetchLedgerRange();
if (!rng)
return {};
if (rng->minSequence == seq)
return KeyIndex{seq};
return indexer_.getKeyIndexOfSeq(seq);
}
BookOffersPage
BackendInterface::fetchBookOffers(
ripple::uint256 const& book,
uint32_t ledgerSequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor) const
{
// TODO try to speed this up. This can take a few seconds. The goal is to
// get it down to a few hundred milliseconds.
BookOffersPage page;
const ripple::uint256 bookEnd = ripple::getQualityNext(book);
ripple::uint256 uTipIndex = book;
bool done = false;
std::vector<ripple::uint256> keys;
auto getMillis = [](auto diff) {
return std::chrono::duration_cast<std::chrono::milliseconds>(diff)
.count();
};
auto begin = std::chrono::system_clock::now();
uint32_t numSucc = 0;
uint32_t numPages = 0;
long succMillis = 0;
long pageMillis = 0;
while (keys.size() < limit)
{
auto mid1 = std::chrono::system_clock::now();
auto offerDir = fetchSuccessor(uTipIndex, ledgerSequence);
auto mid2 = std::chrono::system_clock::now();
numSucc++;
succMillis += getMillis(mid2 - mid1);
if (!offerDir || offerDir->key > bookEnd)
{
BOOST_LOG_TRIVIAL(debug) << __func__ << " - offerDir.has_value() "
<< offerDir.has_value() << " breaking";
break;
}
while (keys.size() < limit)
{
++numPages;
uTipIndex = offerDir->key;
ripple::STLedgerEntry sle{
ripple::SerialIter{
offerDir->blob.data(), offerDir->blob.size()},
offerDir->key};
auto indexes = sle.getFieldV256(ripple::sfIndexes);
keys.insert(keys.end(), indexes.begin(), indexes.end());
// TODO we probably don't have to wait here. We can probably fetch
// these objects in another thread, and move on to another page of
// the book directory, or another directory. We also could just
// accumulate all of the keys before fetching the offers
auto next = sle.getFieldU64(ripple::sfIndexNext);
if (!next)
{
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " next is empty. breaking";
break;
}
auto nextKey = ripple::keylet::page(uTipIndex, next);
auto nextDir = fetchLedgerObject(nextKey.key, ledgerSequence);
assert(nextDir);
offerDir->blob = *nextDir;
offerDir->key = nextKey.key;
}
auto mid3 = std::chrono::system_clock::now();
pageMillis += getMillis(mid3 - mid2);
}
auto mid = std::chrono::system_clock::now();
auto objs = fetchLedgerObjects(keys, ledgerSequence);
for (size_t i = 0; i < keys.size(); ++i)
{
BOOST_LOG_TRIVIAL(trace)
<< __func__ << " key = " << ripple::strHex(keys[i])
<< " blob = " << ripple::strHex(objs[i]);
assert(objs[i].size());
page.offers.push_back({keys[i], objs[i]});
}
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__ << " "
<< "Fetching " << std::to_string(keys.size()) << " keys took "
<< std::to_string(getMillis(mid - begin))
<< " milliseconds. Fetching next dir took "
<< std::to_string(succMillis) << " milliseonds. Fetched next dir "
<< std::to_string(numSucc) << " times"
<< " Fetching next page of dir took " << std::to_string(pageMillis)
<< ". num pages = " << std::to_string(numPages)
<< " milliseconds. Fetching all objects took "
<< std::to_string(getMillis(end - mid))
<< " milliseconds. total time = "
<< std::to_string(getMillis(end - begin)) << " milliseconds";
return page;
}
std::optional<LedgerObject>
BackendInterface::fetchSuccessor(ripple::uint256 key, uint32_t ledgerSequence)
const
{
auto start = std::chrono::system_clock::now();
auto page = fetchLedgerPage({++key}, ledgerSequence, 1, 512);
auto end = std::chrono::system_clock::now();
auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
.count();
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " took " << std::to_string(ms) << " milliseconds";
if (page.objects.size())
return page.objects[0];
return {};
}
LedgerPage
BackendInterface::fetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit,
std::uint32_t limitHint) const
{
assert(limit != 0);
bool incomplete = !isLedgerIndexed(ledgerSequence);
// really low limits almost always miss
uint32_t adjustedLimit = std::max(limitHint, std::max(limit, (uint32_t)4));
LedgerPage page;
page.cursor = cursor;
do
{
adjustedLimit = adjustedLimit >= 8192 ? 8192 : adjustedLimit * 2;
auto start = std::chrono::system_clock::now();
auto partial =
doFetchLedgerPage(page.cursor, ledgerSequence, adjustedLimit);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " " << std::to_string(ledgerSequence) << " "
<< std::to_string(adjustedLimit) << " "
<< ripple::strHex(*page.cursor) << " - time = "
<< std::to_string(
std::chrono::duration_cast<std::chrono::milliseconds>(
end - start)
.count());
page.objects.insert(
page.objects.end(), partial.objects.begin(), partial.objects.end());
page.cursor = partial.cursor;
} while (page.objects.size() < limit && page.cursor);
if (incomplete)
{
auto rng = fetchLedgerRange();
if (!rng)
return page;
if (rng->minSequence == ledgerSequence)
{
BOOST_LOG_TRIVIAL(fatal)
<< __func__
<< " Database is populated but first flag ledger is "
"incomplete. This should never happen";
assert(false);
throw std::runtime_error("Missing base flag ledger");
}
uint32_t lowerSequence = (ledgerSequence - 1) >> indexer_.getKeyShift()
<< indexer_.getKeyShift();
if (lowerSequence < rng->minSequence)
lowerSequence = rng->minSequence;
BOOST_LOG_TRIVIAL(debug)
<< __func__
<< " recursing. ledgerSequence = " << std::to_string(ledgerSequence)
<< " , lowerSequence = " << std::to_string(lowerSequence);
auto lowerPage = fetchLedgerPage(cursor, lowerSequence, limit);
std::vector<ripple::uint256> keys;
std::transform(
std::move_iterator(lowerPage.objects.begin()),
std::move_iterator(lowerPage.objects.end()),
std::back_inserter(keys),
[](auto&& elt) { return std::move(elt.key); });
auto objs = fetchLedgerObjects(keys, ledgerSequence);
for (size_t i = 0; i < keys.size(); ++i)
{
auto& obj = objs[i];
auto& key = keys[i];
if (obj.size())
page.objects.push_back({std::move(key), std::move(obj)});
}
std::sort(page.objects.begin(), page.objects.end(), [](auto a, auto b) {
return a.key < b.key;
});
page.warning = "Data may be incomplete";
}
if (page.objects.size() >= limit)
{
page.objects.resize(limit);
page.cursor = page.objects.back().key;
}
return page;
}
void
BackendInterface::checkFlagLedgers() const
{
auto rng = fetchLedgerRangeNoThrow();
if (rng)
{
bool prevComplete = true;
uint32_t cur = rng->minSequence;
size_t numIncomplete = 0;
while (cur <= rng->maxSequence + 1)
{
auto keyIndex = getKeyIndexOfSeq(cur);
assert(keyIndex.has_value());
cur = keyIndex->keyIndex;
if (!isLedgerIndexed(cur))
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " - flag ledger "
<< std::to_string(keyIndex->keyIndex) << " is incomplete";
++numIncomplete;
prevComplete = false;
}
else
{
if (!prevComplete)
{
BOOST_LOG_TRIVIAL(fatal)
<< __func__ << " - flag ledger "
<< std::to_string(keyIndex->keyIndex)
<< " is incomplete but the next is complete. This "
"should never happen";
assert(false);
throw std::runtime_error("missing prev flag ledger");
}
prevComplete = true;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " - flag ledger "
<< std::to_string(keyIndex->keyIndex) << " is complete";
}
cur = cur + 1;
}
if (numIncomplete > 1)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " " << std::to_string(numIncomplete)
<< " incomplete flag ledgers. "
"This can happen, but is unlikely. Check indexer_key_shift "
"in config";
}
else
{
BOOST_LOG_TRIVIAL(info)
<< __func__ << " number of incomplete flag ledgers = "
<< std::to_string(numIncomplete);
}
}
}
} // namespace Backend

228
reporting/BackendInterface.h
View File

@@ -39,6 +39,8 @@ struct TransactionAndMetadata
Blob transaction;
Blob metadata;
uint32_t ledgerSequence;
bool
operator==(const TransactionAndMetadata&) const = default;
};
struct AccountTransactionsCursor
@@ -53,6 +55,19 @@ struct LedgerRange
uint32_t maxSequence;
};
// The below two structs exist to prevent developers from accidentally mixing up
// the two indexes.
struct BookIndex
{
uint32_t bookIndex;
explicit BookIndex(uint32_t v) : bookIndex(v){};
};
struct KeyIndex
{
uint32_t keyIndex;
explicit KeyIndex(uint32_t v) : keyIndex(v){};
};
class DatabaseTimeout : public std::exception
{
const char*
@@ -65,60 +80,32 @@ class BackendInterface;
class BackendIndexer
{
boost::asio::io_context ioc_;
boost::asio::io_context::strand strand_;
std::mutex mutex_;
std::optional<boost::asio::io_context::work> work_;
std::thread ioThread_;
uint32_t keyShift_ = 20;
uint32_t bookShift_ = 10;
std::unordered_set<ripple::uint256> keys;
std::unordered_set<ripple::uint256> keysCumulative;
std::unordered_map<ripple::uint256, std::unordered_set<ripple::uint256>>
books;
std::unordered_map<ripple::uint256, std::unordered_set<ripple::uint256>>
booksCumulative;
bool populatingCacheAsync = false;
// These are only used when the cache is being populated asynchronously
std::unordered_set<ripple::uint256> deletedKeys;
std::unordered_map<ripple::uint256, std::unordered_set<ripple::uint256>>
deletedBooks;
std::unordered_set<ripple::uint256> keysRepair;
std::unordered_map<ripple::uint256, std::unordered_set<ripple::uint256>>
booksRepair;
std::mutex mtx;
std::condition_variable cv_;
std::atomic_uint32_t indexing_ = 0;
uint32_t keyShift_ = 20;
std::unordered_set<ripple::uint256> keys;
mutable bool isFirst_ = true;
void
addKeyAsync(ripple::uint256 const& key);
doKeysRepair(
BackendInterface const& backend,
std::optional<uint32_t> sequence);
void
addBookOfferAsync(
ripple::uint256 const& book,
ripple::uint256 const& offerKey);
writeKeyFlagLedger(
uint32_t ledgerSequence,
BackendInterface const& backend);
public:
BackendIndexer(boost::json::object const& config);
~BackendIndexer();
void
populateCachesAsync(BackendInterface const& backend);
void
populateCaches(BackendInterface const& backend);
void
clearCaches();
// Blocking, possibly for minutes
void
waitForCaches();
void
addKey(ripple::uint256 const& key);
void
deleteKey(ripple::uint256 const& key);
void
addBookOffer(ripple::uint256 const& book, ripple::uint256 const& offerKey);
void
deleteBookOffer(
ripple::uint256 const& book,
ripple::uint256 const& offerKey);
addKey(ripple::uint256&& key);
void
finish(uint32_t ledgerSequence, BackendInterface const& backend);
@@ -127,59 +114,44 @@ public:
uint32_t ledgerSequence,
BackendInterface const& backend);
void
writeBookFlagLedgerAsync(
uint32_t ledgerSequence,
BackendInterface const& backend);
void
doKeysRepair(
doKeysRepairAsync(
BackendInterface const& backend,
std::optional<uint32_t> sequence);
void
doBooksRepair(
BackendInterface const& backend,
std::optional<uint32_t> sequence);
uint32_t
getBookShift()
{
return bookShift_;
}
uint32_t
getKeyShift()
{
return keyShift_;
}
uint32_t
std::optional<uint32_t>
getCurrentlyIndexing()
{
uint32_t cur = indexing_.load();
if (cur != 0)
return cur;
return {};
}
KeyIndex
getKeyIndexOfSeq(uint32_t seq) const
{
if (isKeyFlagLedger(seq))
return seq;
return KeyIndex{seq};
auto incr = (1 << keyShift_);
return (seq >> keyShift_ << keyShift_) + incr;
KeyIndex index{(seq >> keyShift_ << keyShift_) + incr};
assert(isKeyFlagLedger(index.keyIndex));
return index;
}
bool
isKeyFlagLedger(uint32_t ledgerSequence) const
{
return (ledgerSequence % (1 << keyShift_)) == 0;
}
uint32_t
getBookIndexOfSeq(uint32_t seq) const
{
if (isBookFlagLedger(seq))
return seq;
auto incr = (1 << bookShift_);
return (seq >> bookShift_ << bookShift_) + incr;
}
bool
isBookFlagLedger(uint32_t ledgerSequence) const
{
return (ledgerSequence % (1 << bookShift_)) == 0;
}
};
class BackendInterface
{
protected:
mutable BackendIndexer indexer_;
mutable bool isFirst_ = true;
public:
// read methods
@@ -193,45 +165,14 @@ public:
return indexer_;
}
std::optional<uint32_t>
getKeyIndexOfSeq(uint32_t seq) const
{
if (indexer_.isKeyFlagLedger(seq))
return seq;
auto rng = fetchLedgerRange();
if (!rng)
return {};
if (rng->minSequence == seq)
return seq;
return indexer_.getKeyIndexOfSeq(seq);
}
std::optional<uint32_t>
getBookIndexOfSeq(uint32_t seq) const
{
if (indexer_.isBookFlagLedger(seq))
return seq;
auto rng = fetchLedgerRange();
if (!rng)
return {};
if (rng->minSequence == seq)
return seq;
return indexer_.getBookIndexOfSeq(seq);
}
void
checkFlagLedgers() const;
std::optional<KeyIndex>
getKeyIndexOfSeq(uint32_t seq) const;
bool
finishWrites(uint32_t ledgerSequence) const
{
indexer_.finish(ledgerSequence, *this);
auto commitRes = doFinishWrites();
if (commitRes)
{
if (indexer_.isBookFlagLedger(ledgerSequence))
indexer_.writeBookFlagLedgerAsync(ledgerSequence, *this);
if (indexer_.isKeyFlagLedger(ledgerSequence))
indexer_.writeKeyFlagLedgerAsync(ledgerSequence, *this);
}
return commitRes;
}
finishWrites(uint32_t ledgerSequence) const;
virtual std::optional<uint32_t>
fetchLatestLedgerSequence() const = 0;
@@ -243,20 +184,7 @@ public:
fetchLedgerRange() const = 0;
std::optional<LedgerRange>
fetchLedgerRangeNoThrow() const
{
while (true)
{
try
{
return fetchLedgerRange();
}
catch (DatabaseTimeout& t)
{
;
}
}
}
fetchLedgerRangeNoThrow() const;
virtual std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence) const = 0;
@@ -271,19 +199,32 @@ public:
virtual std::vector<ripple::uint256>
fetchAllTransactionHashesInLedger(uint32_t ledgerSequence) const = 0;
virtual LedgerPage
LedgerPage
fetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit,
std::uint32_t limitHint = 0) const;
bool
isLedgerIndexed(std::uint32_t ledgerSequence) const;
std::optional<LedgerObject>
fetchSuccessor(ripple::uint256 key, uint32_t ledgerSequence) const;
virtual LedgerPage
doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const = 0;
// TODO add warning for incomplete data
virtual BookOffersPage
BookOffersPage
fetchBookOffers(
ripple::uint256 const& book,
uint32_t ledgerSequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor = {}) const = 0;
std::optional<ripple::uint256> const& cursor = {}) const;
virtual std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes) const = 0;
@@ -316,28 +257,8 @@ public:
std::string&& blob,
bool isCreated,
bool isDeleted,
std::optional<ripple::uint256>&& book) const
{
ripple::uint256 key256 = ripple::uint256::fromVoid(key.data());
if (isCreated)
indexer_.addKey(key256);
if (isDeleted)
indexer_.deleteKey(key256);
if (book)
{
if (isCreated)
indexer_.addBookOffer(*book, key256);
if (isDeleted)
indexer_.deleteBookOffer(*book, key256);
}
doWriteLedgerObject(
std::move(key),
seq,
std::move(blob),
isCreated,
isDeleted,
std::move(book));
}
std::optional<ripple::uint256>&& book) const;
virtual void
doWriteLedgerObject(
std::string&& key,
@@ -377,18 +298,11 @@ public:
doFinishWrites() const = 0;
virtual bool
doOnlineDelete(uint32_t minLedgerToKeep) const = 0;
doOnlineDelete(uint32_t numLedgersToKeep) const = 0;
virtual bool
writeKeys(
std::unordered_set<ripple::uint256> const& keys,
uint32_t ledgerSequence,
bool isAsync = false) const = 0;
virtual bool
writeBooks(
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books,
uint32_t ledgerSequence,
KeyIndex const& index,
bool isAsync = false) const = 0;
virtual ~BackendInterface()

499
reporting/CassandraBackend.cpp
View File

@@ -394,7 +394,7 @@ CassandraBackend::fetchLedgerDiff(uint32_t ledgerSequence) const
return objects;
}
LedgerPage
CassandraBackend::fetchLedgerPage(
CassandraBackend::doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const
@@ -405,12 +405,12 @@ CassandraBackend::fetchLedgerPage(
LedgerPage page;
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " ledgerSequence = " << std::to_string(ledgerSequence)
<< " index = " << std::to_string(*index);
<< " index = " << std::to_string(index->keyIndex);
if (cursor)
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " - Cursor = " << ripple::strHex(*cursor);
CassandraStatement statement{selectKeys_};
statement.bindInt(*index);
statement.bindInt(index->keyIndex);
if (cursor)
statement.bindBytes(*cursor);
else
@@ -422,7 +422,7 @@ CassandraBackend::fetchLedgerPage(
CassandraResult result = executeSyncRead(statement);
if (!!result)
{
BOOST_LOG_TRIVIAL(trace)
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " - got keys - size = " << result.numRows();
std::vector<ripple::uint256> keys;
@@ -430,17 +430,17 @@ CassandraBackend::fetchLedgerPage(
{
keys.push_back(result.getUInt256());
} while (result.nextRow());
if (keys.size() && keys.size() == limit)
if (keys.size() && keys.size() >= limit)
{
page.cursor = keys.back();
keys.pop_back();
++(*page.cursor);
}
auto objects = fetchLedgerObjects(keys, ledgerSequence);
if (objects.size() != keys.size())
throw std::runtime_error("Mismatch in size of objects and keys");
if (cursor)
BOOST_LOG_TRIVIAL(trace)
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " Cursor = " << ripple::strHex(*page.cursor);
for (size_t i = 0; i < objects.size(); ++i)
@@ -494,129 +494,6 @@ CassandraBackend::fetchLedgerObjects(
<< "Fetched " << numKeys << " records from Cassandra";
return results;
}
BookOffersPage
CassandraBackend::fetchBookOffers(
ripple::uint256 const& book,
uint32_t ledgerSequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor) const
{
auto rng = fetchLedgerRange();
auto limitTuningFactor = 50;
if(!rng)
return {{},{}};
auto readBooks =
[this, &book, &limit, &limitTuningFactor]
(std::uint32_t sequence)
-> std::pair<bool, std::vector<std::pair<std::uint64_t, ripple::uint256>>>
{
CassandraStatement completeQuery{completeBook_};
completeQuery.bindInt(sequence);
CassandraResult completeResult = executeSyncRead(completeQuery);
bool complete = completeResult.hasResult();
CassandraStatement statement{selectBook_};
std::vector<std::pair<std::uint64_t, ripple::uint256>> keys = {};
statement.bindBytes(book.data(), 24);
statement.bindInt(sequence);
BOOST_LOG_TRIVIAL(info) << __func__ << " upper = " << std::to_string(sequence)
<< " book = " << ripple::strHex(std::string((char*)book.data(), 24));
ripple::uint256 zero = beast::zero;
statement.bindBytes(zero.data(), 8);
statement.bindBytes(zero);
statement.bindUInt(limit * limitTuningFactor);
auto start = std::chrono::system_clock::now();
CassandraResult result = executeSyncRead(statement);
auto end = std::chrono::system_clock::now();
auto duration = ((end - start).count()) / 1000000000.0;
BOOST_LOG_TRIVIAL(info) << "Book directory fetch took "
<< std::to_string(duration) << " seconds.";
BOOST_LOG_TRIVIAL(debug) << __func__ << " - got keys";
if (!result)
{
return {false, {{}, {}}};
}
do
{
auto [quality, index] = result.getBytesTuple();
std::uint64_t q = 0;
memcpy(&q, quality.data(), 8);
keys.push_back({q, ripple::uint256::fromVoid(index.data())});
} while (result.nextRow());
return {complete, keys};
};
auto upper = indexer_.getBookIndexOfSeq(ledgerSequence);
auto [complete, quality_keys] = readBooks(upper);
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " - populated keys. num keys = " << quality_keys.size();
std::optional<std::string> warning = {};
if (!complete)
{
warning = "Data may be incomplete";
BOOST_LOG_TRIVIAL(info) << "May be incomplete. Fetching other page";
auto bookShift = indexer_.getBookShift();
std::uint32_t lower = upper - (1 << bookShift);
auto originalKeys = std::move(quality_keys);
auto [lowerComplete, otherKeys] = readBooks(lower);
assert(lowerComplete);
std::vector<std::pair<std::uint64_t, ripple::uint256>> merged_keys;
merged_keys.reserve(originalKeys.size() + otherKeys.size());
std::merge(originalKeys.begin(), originalKeys.end(),
otherKeys.begin(), otherKeys.end(),
std::back_inserter(merged_keys),
[](auto pair1, auto pair2)
{
return pair1.first < pair2.first;
});
}
std::vector<ripple::uint256> merged(quality_keys.size());
std::transform(quality_keys.begin(), quality_keys.end(),
std::back_inserter(merged),
[](auto pair) { return pair.second; });
auto uniqEnd = std::unique(merged.begin(), merged.end());
std::vector<ripple::uint256> keys{merged.begin(), uniqEnd};
std::cout << keys.size() << std::endl;
auto start = std::chrono::system_clock::now();
std::vector<Blob> objs = fetchLedgerObjects(keys, ledgerSequence);
auto end = std::chrono::system_clock::now();
auto duration = ((end - start).count()) / 1000000000.0;
BOOST_LOG_TRIVIAL(info) << "Book object fetch took "
<< std::to_string(duration) << " seconds.";
std::vector<LedgerObject> results;
for (size_t i = 0; i < objs.size(); ++i)
{
if (objs[i].size() != 0)
results.push_back({keys[i], objs[i]});
}
return {results, {}, warning};
}
struct WriteBookCallbackData
{
CassandraBackend const& backend;
@@ -654,7 +531,7 @@ writeBook(WriteBookCallbackData& cb)
CassandraStatement statement{cb.backend.getInsertBookPreparedStatement()};
statement.bindBytes(cb.book.data(), 24);
statement.bindInt(cb.ledgerSequence);
statement.bindBytes(cb.book.data()+24, 8);
statement.bindBytes(cb.book.data() + 24, 8);
statement.bindBytes(cb.offerKey);
// Passing isRetry as true bypasses incrementing numOutstanding
cb.backend.executeAsyncWrite(statement, writeBookCallback, cb, true);
@@ -723,6 +600,87 @@ struct WriteKeyCallbackData
{
}
};
struct OnlineDeleteCallbackData
{
CassandraBackend const& backend;
ripple::uint256 key;
uint32_t ledgerSequence;
std::vector<unsigned char> object;
std::condition_variable& cv;
std::atomic_uint32_t& numOutstanding;
std::mutex& mtx;
uint32_t currentRetries = 0;
OnlineDeleteCallbackData(
CassandraBackend const& backend,
ripple::uint256&& key,
uint32_t ledgerSequence,
std::vector<unsigned char>&& object,
std::condition_variable& cv,
std::mutex& mtx,
std::atomic_uint32_t& numOutstanding)
: backend(backend)
, key(std::move(key))
, ledgerSequence(ledgerSequence)
, object(std::move(object))
, cv(cv)
, mtx(mtx)
, numOutstanding(numOutstanding)
{
}
};
void
onlineDeleteCallback(CassFuture* fut, void* cbData);
void
onlineDelete(OnlineDeleteCallbackData& cb)
{
{
CassandraStatement statement{
cb.backend.getInsertObjectPreparedStatement()};
statement.bindBytes(cb.key);
statement.bindInt(cb.ledgerSequence);
statement.bindBytes(cb.object);
cb.backend.executeAsyncWrite(statement, onlineDeleteCallback, cb, true);
}
}
void
onlineDeleteCallback(CassFuture* fut, void* cbData)
{
OnlineDeleteCallbackData& requestParams =
*static_cast<OnlineDeleteCallbackData*>(cbData);
CassandraBackend const& backend = requestParams.backend;
auto rc = cass_future_error_code(fut);
if (rc != CASS_OK)
{
// exponential backoff with a max wait of 2^10 ms (about 1 second)
auto wait = std::chrono::milliseconds(
lround(std::pow(2, std::min(10u, requestParams.currentRetries))));
BOOST_LOG_TRIVIAL(error)
<< "ERROR!!! Cassandra insert book error: " << rc << ", "
<< cass_error_desc(rc) << ", retrying in " << wait.count()
<< " milliseconds";
++requestParams.currentRetries;
std::shared_ptr<boost::asio::steady_timer> timer =
std::make_shared<boost::asio::steady_timer>(
backend.getIOContext(),
std::chrono::steady_clock::now() + wait);
timer->async_wait(
[timer, &requestParams](const boost::system::error_code& error) {
onlineDelete(requestParams);
});
}
else
{
BOOST_LOG_TRIVIAL(trace) << __func__ << " Successfully inserted a book";
{
std::lock_guard lck(requestParams.mtx);
--requestParams.numOutstanding;
requestParams.cv.notify_one();
}
}
}
void
writeKeyCallback(CassFuture* fut, void* cbData);
void
@@ -775,14 +733,9 @@ writeKeyCallback(CassFuture* fut, void* cbData)
bool
CassandraBackend::writeKeys(
std::unordered_set<ripple::uint256> const& keys,
uint32_t ledgerSequence,
KeyIndex const& index,
bool isAsync) const
{
BOOST_LOG_TRIVIAL(info)
<< __func__ << " Ledger = " << std::to_string(ledgerSequence)
<< " . num keys = " << std::to_string(keys.size())
<< " . concurrentLimit = "
<< std::to_string(indexerMaxRequestsOutstanding);
std::atomic_uint32_t numRemaining = keys.size();
std::condition_variable cv;
std::mutex mtx;
@@ -790,11 +743,16 @@ CassandraBackend::writeKeys(
cbs.reserve(keys.size());
uint32_t concurrentLimit =
isAsync ? indexerMaxRequestsOutstanding : keys.size();
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " Ledger = " << std::to_string(index.keyIndex)
<< " . num keys = " << std::to_string(keys.size())
<< " . concurrentLimit = "
<< std::to_string(indexerMaxRequestsOutstanding);
uint32_t numSubmitted = 0;
for (auto& key : keys)
{
cbs.push_back(std::make_shared<WriteKeyCallbackData>(
*this, key, ledgerSequence, cv, mtx, numRemaining));
*this, key, index.keyIndex, cv, mtx, numRemaining));
writeKey(*cbs.back());
++numSubmitted;
BOOST_LOG_TRIVIAL(trace) << __func__ << "Submitted a write request";
@@ -812,7 +770,7 @@ CassandraBackend::writeKeys(
concurrentLimit;
});
if (numSubmitted % 100000 == 0)
BOOST_LOG_TRIVIAL(info)
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " Submitted " << std::to_string(numSubmitted)
<< " write requests. Completed "
<< (keys.size() - numRemaining);
@@ -823,57 +781,6 @@ CassandraBackend::writeKeys(
return true;
}
bool
CassandraBackend::writeBooks(
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books,
uint32_t ledgerSequence,
bool isAsync) const
{
BOOST_LOG_TRIVIAL(info)
<< __func__ << " Ledger = " << std::to_string(ledgerSequence)
<< " . num books = " << std::to_string(books.size());
std::condition_variable cv;
std::mutex mtx;
std::vector<std::shared_ptr<WriteBookCallbackData>> cbs;
uint32_t concurrentLimit =
isAsync ? indexerMaxRequestsOutstanding : maxRequestsOutstanding;
std::atomic_uint32_t numOutstanding = 0;
size_t count = 0;
auto start = std::chrono::system_clock::now();
for (auto& book : books)
{
for (auto& offer : book.second)
{
++numOutstanding;
++count;
cbs.push_back(std::make_shared<WriteBookCallbackData>(
*this,
book.first,
offer,
ledgerSequence,
cv,
mtx,
numOutstanding));
writeBook(*cbs.back());
BOOST_LOG_TRIVIAL(trace) << __func__ << "Submitted a write request";
std::unique_lock<std::mutex> lck(mtx);
BOOST_LOG_TRIVIAL(trace) << __func__ << "Got the mutex";
cv.wait(lck, [&numOutstanding, concurrentLimit]() {
return numOutstanding < concurrentLimit;
});
}
}
BOOST_LOG_TRIVIAL(info) << __func__
<< "Submitted all book writes. Waiting for them to "
"finish. num submitted = "
<< std::to_string(count);
std::unique_lock<std::mutex> lck(mtx);
cv.wait(lck, [&numOutstanding]() { return numOutstanding == 0; });
BOOST_LOG_TRIVIAL(info) << __func__ << "Finished writing books";
return true;
}
bool
CassandraBackend::isIndexed(uint32_t ledgerSequence) const
{
@@ -1100,10 +1007,79 @@ CassandraBackend::runIndexer(uint32_t ledgerSequence) const
*/
}
bool
CassandraBackend::doOnlineDelete(uint32_t minLedgerToKeep) const
CassandraBackend::doOnlineDelete(uint32_t numLedgersToKeep) const
{
throw std::runtime_error("doOnlineDelete : unimplemented");
return false;
// calculate TTL
// ledgers close roughly every 4 seconds. We double the TTL so that way
// there is a window of time to update the database, to prevent unchanging
// records from being deleted.
auto rng = fetchLedgerRangeNoThrow();
if (!rng)
return false;
uint32_t minLedger = rng->maxSequence - numLedgersToKeep;
if (minLedger <= rng->minSequence)
return false;
std::condition_variable cv;
std::mutex mtx;
std::vector<std::shared_ptr<OnlineDeleteCallbackData>> cbs;
uint32_t concurrentLimit = 10;
std::atomic_uint32_t numOutstanding = 0;
// iterate through latest ledger, updating TTL
std::optional<ripple::uint256> cursor;
while (true)
{
try
{
auto [objects, curCursor, warning] =
fetchLedgerPage(cursor, minLedger, 256);
if (warning)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__
<< " online delete running but flag ledger is not complete";
std::this_thread::sleep_for(std::chrono::seconds(10));
continue;
}
for (auto& obj : objects)
{
++numOutstanding;
cbs.push_back(std::make_shared<OnlineDeleteCallbackData>(
*this,
std::move(obj.key),
minLedger,
std::move(obj.blob),
cv,
mtx,
numOutstanding));
onlineDelete(*cbs.back());
std::unique_lock<std::mutex> lck(mtx);
BOOST_LOG_TRIVIAL(trace) << __func__ << "Got the mutex";
cv.wait(lck, [&numOutstanding, concurrentLimit]() {
return numOutstanding < concurrentLimit;
});
}
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
cursor = curCursor;
if (!cursor)
break;
}
catch (DatabaseTimeout const& e)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " Database timeout fetching keys";
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
std::unique_lock<std::mutex> lck(mtx);
cv.wait(lck, [&numOutstanding]() { return numOutstanding == 0; });
CassandraStatement statement{deleteLedgerRange_};
statement.bindInt(minLedger);
executeSyncWrite(statement);
// update ledger_range
return true;
}
void
@@ -1117,6 +1093,11 @@ CassandraBackend::open(bool readOnly)
}
return {""};
};
auto getInt = [this](std::string const& field) -> std::optional<int> {
if (config_.contains(field) && config_.at(field).is_int64())
return config_[field].as_int64();
return {};
};
if (open_)
{
assert(false);
@@ -1169,14 +1150,14 @@ CassandraBackend::open(bool readOnly)
throw std::runtime_error(ss.str());
}
int port = config_.contains("port") ? config_["port"].as_int64() : 0;
auto port = getInt("port");
if (port)
{
rc = cass_cluster_set_port(cluster, port);
rc = cass_cluster_set_port(cluster, *port);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error setting Cassandra port: " << port
ss << "nodestore: Error setting Cassandra port: " << *port
<< ", result: " << rc << ", " << cass_error_desc(rc);
throw std::runtime_error(ss.str());
@@ -1204,9 +1185,8 @@ CassandraBackend::open(bool readOnly)
cass_cluster_set_credentials(
cluster, username.c_str(), getString("password").c_str());
}
int threads = config_.contains("threads")
? config_["threads"].as_int64()
: std::thread::hardware_concurrency();
int threads = getInt("threads") ? *getInt("threads")
: std::thread::hardware_concurrency();
rc = cass_cluster_set_num_threads_io(cluster, threads);
if (rc != CASS_OK)
@@ -1216,6 +1196,8 @@ CassandraBackend::open(bool readOnly)
<< ", result: " << rc << ", " << cass_error_desc(rc);
throw std::runtime_error(ss.str());
}
if (getInt("max_requests_outstanding"))
maxRequestsOutstanding = *getInt("max_requests_outstanding");
cass_cluster_set_request_timeout(cluster, 10000);
@@ -1272,10 +1254,13 @@ CassandraBackend::open(bool readOnly)
std::string keyspace = getString("keyspace");
if (keyspace.empty())
{
throw std::runtime_error(
"nodestore: Missing keyspace in Cassandra config");
BOOST_LOG_TRIVIAL(warning)
<< "No keyspace specified. Using keyspace oceand";
keyspace = "oceand";
}
int rf = getInt("replication_factor") ? *getInt("replication_factor") : 3;
std::string tablePrefix = getString("table_prefix");
if (tablePrefix.empty())
{
@@ -1284,6 +1269,19 @@ CassandraBackend::open(bool readOnly)
cass_cluster_set_connect_timeout(cluster, 10000);
int ttl = getInt("ttl") ? *getInt("ttl") * 2 : 0;
int keysTtl = (ttl != 0 ? pow(2, indexer_.getKeyShift()) * 4 * 2 : 0);
int incr = keysTtl;
while (keysTtl < ttl)
{
keysTtl += incr;
}
int booksTtl = 0;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " setting ttl to " << std::to_string(ttl)
<< " , books ttl to " << std::to_string(booksTtl) << " , keys ttl to "
<< std::to_string(keysTtl);
auto executeSimpleStatement = [this](std::string const& query) {
CassStatement* statement = makeStatement(query.c_str(), 0);
CassFuture* fut = cass_session_execute(session_.get(), statement);
@@ -1317,8 +1315,36 @@ CassandraBackend::open(bool readOnly)
{
std::stringstream ss;
ss << "nodestore: Error connecting Cassandra session keyspace: "
<< rc << ", " << cass_error_desc(rc);
<< rc << ", " << cass_error_desc(rc)
<< ", trying to create it ourselves";
BOOST_LOG_TRIVIAL(error) << ss.str();
// if the keyspace doesn't exist, try to create it
session_.reset(cass_session_new());
fut = cass_session_connect(session_.get(), cluster);
rc = cass_future_error_code(fut);
cass_future_free(fut);
if (rc != CASS_OK)
{
std::stringstream ss;
ss << "nodestore: Error connecting Cassandra session at all: "
<< rc << ", " << cass_error_desc(rc);
BOOST_LOG_TRIVIAL(error) << ss.str();
}
else
{
std::stringstream query;
query << "CREATE KEYSPACE IF NOT EXISTS " << keyspace
<< " WITH replication = {'class': 'SimpleStrategy', "
"'replication_factor': '"
<< std::to_string(rf) << "'} AND durable_writes = true";
if (!executeSimpleStatement(query.str()))
continue;
query = {};
query << "USE " << keyspace;
if (!executeSimpleStatement(query.str()))
continue;
}
continue;
}
@@ -1326,7 +1352,8 @@ CassandraBackend::open(bool readOnly)
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "objects"
<< " ( key blob, sequence bigint, object blob, PRIMARY "
"KEY(key, "
"sequence)) WITH CLUSTERING ORDER BY (sequence DESC)";
"sequence)) WITH CLUSTERING ORDER BY (sequence DESC) AND"
<< " default_time_to_live = " << std::to_string(ttl);
if (!executeSimpleStatement(query.str()))
continue;
@@ -1337,6 +1364,7 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
<<<<<<< HEAD
query << "CREATE INDEX ON " << tablePrefix << "objects(sequence)";
if (!executeSimpleStatement(query.str()))
continue;
@@ -1352,6 +1380,13 @@ CassandraBackend::open(bool readOnly)
<< "CREATE TABLE IF NOT EXISTS " << tablePrefix << "transactions"
<< " ( hash blob PRIMARY KEY, ledger_sequence bigint, transaction "
"blob, metadata blob)";
=======
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "transactions"
<< " ( hash blob PRIMARY KEY, ledger_sequence bigint, "
"transaction "
"blob, metadata blob)"
<< " WITH default_time_to_live = " << std::to_string(ttl);
>>>>>>> dev
if (!executeSimpleStatement(query.str()))
continue;
@@ -1376,7 +1411,9 @@ CassandraBackend::open(bool readOnly)
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "keys"
<< " ( sequence bigint, key blob, PRIMARY KEY "
"(sequence, key))";
"(sequence, key))"
" WITH default_time_to_live = "
<< std::to_string(keysTtl);
if (!executeSimpleStatement(query.str()))
continue;
@@ -1386,24 +1423,14 @@ CassandraBackend::open(bool readOnly)
if (!executeSimpleStatement(query.str()))
continue;
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "books"
<< " ( book blob, sequence bigint, quality_key tuple<blob, blob>, PRIMARY KEY "
"((book, sequence), quality_key)) WITH CLUSTERING ORDER BY (quality_key "
"ASC)";
if (!executeSimpleStatement(query.str()))
continue;
query.str("");
query << "SELECT * FROM " << tablePrefix << "books"
<< " LIMIT 1";
if (!executeSimpleStatement(query.str()))
continue;
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "account_tx"
<< " ( account blob, seq_idx tuple<bigint, bigint>, "
" hash blob, "
"PRIMARY KEY "
"(account, seq_idx)) WITH "
"CLUSTERING ORDER BY (seq_idx desc)";
"CLUSTERING ORDER BY (seq_idx desc)"
<< " AND default_time_to_live = " << std::to_string(ttl);
if (!executeSimpleStatement(query.str()))
continue;
@@ -1415,7 +1442,8 @@ CassandraBackend::open(bool readOnly)
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "ledgers"
<< " ( sequence bigint PRIMARY KEY, header blob )";
<< " ( sequence bigint PRIMARY KEY, header blob )"
<< " WITH default_time_to_live = " << std::to_string(ttl);
if (!executeSimpleStatement(query.str()))
continue;
@@ -1427,7 +1455,8 @@ CassandraBackend::open(bool readOnly)
query.str("");
query << "CREATE TABLE IF NOT EXISTS " << tablePrefix << "ledger_hashes"
<< " (hash blob PRIMARY KEY, sequence bigint)";
<< " (hash blob PRIMARY KEY, sequence bigint)"
<< " WITH default_time_to_live = " << std::to_string(ttl);
if (!executeSimpleStatement(query.str()))
continue;
@@ -1478,12 +1507,15 @@ CassandraBackend::open(bool readOnly)
continue;
query.str("");
<<<<<<< HEAD
query << "INSERT INTO " << tablePrefix << "books"
<< " (book, sequence, quality_key) VALUES (?, ?, (?, ?))";
if (!insertBook2_.prepareStatement(query, session_.get()))
continue;
query.str("");
=======
>>>>>>> dev
query << "SELECT key FROM " << tablePrefix << "keys"
<< " WHERE sequence = ? AND key >= ? ORDER BY key ASC LIMIT ?";
if (!selectKeys_.prepareStatement(query, session_.get()))
@@ -1541,24 +1573,6 @@ CassandraBackend::open(bool readOnly)
if (!getToken_.prepareStatement(query, session_.get()))
continue;
query.str("");
query << "SELECT quality_key FROM " << tablePrefix << "books "
<< " WHERE book = ? AND sequence = ?"
<< " AND quality_key >= (?, ?)"
" ORDER BY quality_key ASC "
" LIMIT ?";
if (!selectBook_.prepareStatement(query, session_.get()))
continue;
query.str("");
query << "SELECT * FROM " << tablePrefix << "books "
<< "WHERE book = "
<< "0x000000000000000000000000000000000000000000000000"
<< " AND sequence = ?";
if (!completeBook_.prepareStatement(query, session_.get()))
continue;
query.str("");
query << " INSERT INTO " << tablePrefix << "account_tx"
<< " (account, seq_idx, hash) "
@@ -1591,6 +1605,11 @@ CassandraBackend::open(bool readOnly)
"(?,null)";
if (!updateLedgerRange_.prepareStatement(query, session_.get()))
continue;
query = {};
query << " update " << tablePrefix << "ledger_range"
<< " set sequence = ? where is_latest = false";
if (!deleteLedgerRange_.prepareStatement(query, session_.get()))
continue;
query.str("");
query << " select header from " << tablePrefix
@@ -1610,15 +1629,17 @@ CassandraBackend::open(bool readOnly)
<< " is_latest IN (true, false)";
if (!selectLedgerRange_.prepareStatement(query, session_.get()))
continue;
/*
query.str("");
query << " SELECT key,object FROM " << tablePrefix
<< "objects WHERE sequence = ?";
if (!selectLedgerDiff_.prepareStatement(query, session_.get()))
continue;
*/
setupPreparedStatements = true;
}
<<<<<<< HEAD
if (config_.contains("max_requests_outstanding"))
{
maxRequestsOutstanding = config_["max_requests_outstanding"].as_int64();
@@ -1629,6 +1650,8 @@ CassandraBackend::open(bool readOnly)
config_["indexer_max_requests_outstanding"].as_int64();
}
=======
>>>>>>> dev
work_.emplace(ioContext_);
ioThread_ = std::thread{[this]() { ioContext_.run(); }};
open_ = true;

40
reporting/CassandraBackend.h
View File

@@ -166,6 +166,11 @@ public:
bindBytes(data.data(), data.size());
}
void
bindBytes(std::vector<unsigned char> const& data)
{
bindBytes(data.data(), data.size());
}
void
bindBytes(ripple::AccountID const& data)
{
bindBytes(data.data(), data.size());
@@ -649,6 +654,7 @@ private:
CassandraPreparedStatement insertLedgerHeader_;
CassandraPreparedStatement insertLedgerHash_;
CassandraPreparedStatement updateLedgerRange_;
CassandraPreparedStatement deleteLedgerRange_;
CassandraPreparedStatement updateLedgerHeader_;
CassandraPreparedStatement selectLedgerBySeq_;
CassandraPreparedStatement selectLatestLedger_;
@@ -735,6 +741,11 @@ public:
{
return insertBook2_;
}
CassandraPreparedStatement const&
getInsertObjectPreparedStatement() const
{
return insertObject_;
}
CassandraPreparedStatement const&
getSelectLedgerDiffPreparedStatement() const
@@ -830,14 +841,6 @@ public:
{
// wait for all other writes to finish
sync();
auto rng = fetchLedgerRangeNoThrow();
if (rng && rng->maxSequence >= ledgerSequence_)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " Ledger " << std::to_string(ledgerSequence_)
<< " already written. Returning";
return false;
}
// write range
if (isFirstLedger_)
{
@@ -954,7 +957,7 @@ public:
CassandraResult result = executeSyncRead(statement);
if (!result)
{
BOOST_LOG_TRIVIAL(error) << __func__ << " - no rows";
BOOST_LOG_TRIVIAL(debug) << __func__ << " - no rows";
return {};
}
return result.getBytes();
@@ -994,7 +997,7 @@ public:
return {{result.getBytes(), result.getBytes(), result.getUInt32()}};
}
LedgerPage
fetchLedgerPage(
doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const override;
@@ -1014,21 +1017,8 @@ public:
bool
writeKeys(
std::unordered_set<ripple::uint256> const& keys,
uint32_t ledgerSequence,
bool isAsync = false) const;
bool
writeBooks(
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books,
uint32_t ledgerSequence,
KeyIndex const& index,
bool isAsync = false) const override;
BookOffersPage
fetchBookOffers(
ripple::uint256 const& book,
uint32_t sequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor) const override;
bool
canFetchBatch()
@@ -1353,7 +1343,7 @@ public:
syncCv_.wait(lck, [this]() { return finishedAllRequests(); });
}
bool
doOnlineDelete(uint32_t minLedgerToKeep) const override;
doOnlineDelete(uint32_t numLedgersToKeep) const override;
boost::asio::io_context&
getIOContext() const

2
reporting/DBHelpers.h
View File

@@ -44,6 +44,8 @@ struct AccountTransactionsData
, txHash(txHash)
{
}
AccountTransactionsData() = default;
};
template <class T>

61
reporting/ETLSource.h
View File

@@ -253,10 +253,29 @@ public:
", grpc port : " + grpcPort_ + " }";
}
<<<<<<< HEAD
boost::json::value
toJson() const
{
return boost::json::string(toString());
=======
boost::json::object
toJson() const
{
boost::json::object res;
res["validated_range"] = getValidatedRange();
res["is_connected"] = std::to_string(isConnected());
res["ip"] = ip_;
res["ws_port"] = wsPort_;
res["grpc_port"] = grpcPort_;
auto last = getLastMsgTime();
if (last.time_since_epoch().count() != 0)
res["last_msg_arrival_time"] = std::to_string(
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - getLastMsgTime())
.count());
return res;
>>>>>>> dev
}
/// Download a ledger in full
@@ -377,6 +396,7 @@ public:
/// to clients).
/// @param in ETLSource in question
/// @return true if messages should be forwarded
<<<<<<< HEAD
bool
shouldPropagateTxnStream(ETLSource* in) const
{
@@ -398,11 +418,35 @@ public:
}
boost::json::value
=======
// bool
// shouldPropagateTxnStream(ETLSource* in) const
// {
// for (auto& src : sources_)
// {
// assert(src);
// // We pick the first ETLSource encountered that is connected
// if (src->isConnected())
// {
// if (src.get() == in)
// return true;
// else
// return false;
// }
// }
//
// // If no sources connected, then this stream has not been
// forwarded. return true;
// }
boost::json::array
>>>>>>> dev
toJson() const
{
boost::json::array ret;
for (auto& src : sources_)
{
<<<<<<< HEAD
ret.push_back(src->toJson());
}
return ret;
@@ -418,6 +462,23 @@ public:
/// @return response received from p2p node
boost::json::object
forwardToP2p(boost::json::object const& request) const;
=======
ret.emplace_back(src->toJson());
}
return ret;
}
//
// /// Randomly select a p2p node to forward a gRPC request to
// /// @return gRPC stub to forward requests to p2p node
// std::unique_ptr<org::xrpl::rpc::v1::XRPLedgerAPIService::Stub>
// getP2pForwardingStub() const;
//
// /// Forward a JSON RPC request to a randomly selected p2p node
// /// @param context context of the request
// /// @return response received from p2p node
// Json::Value
// forwardToP2p(RPC::JsonContext& context) const;
>>>>>>> dev
private:
/// f is a function that takes an ETLSource as an argument and returns a

85
reporting/Pg.cpp
View File

@@ -40,6 +40,7 @@
#include <functional>
#include <iterator>
#include <reporting/Pg.h>
#include <signal.h>
#include <sstream>
#include <stdexcept>
#include <string>
@@ -47,12 +48,11 @@
#include <thread>
#include <utility>
#include <vector>
#include <signal.h>
static void
noticeReceiver(void* arg, PGresult const* res)
{
BOOST_LOG_TRIVIAL(debug) << "server message: " << PQresultErrorMessage(res);
BOOST_LOG_TRIVIAL(trace) << "server message: " << PQresultErrorMessage(res);
}
//-----------------------------------------------------------------------------
@@ -242,8 +242,10 @@ Pg::bulkInsert(char const* table, std::string const& records)
{
// https://www.postgresql.org/docs/12/libpq-copy.html#LIBPQ-COPY-SEND
assert(conn_.get());
static auto copyCmd = boost::format(R"(COPY %s FROM stdin)");
auto res = query(boost::str(copyCmd % table).c_str());
auto copyCmd = boost::format(R"(COPY %s FROM stdin)");
auto formattedCmd = boost::str(copyCmd % table);
BOOST_LOG_TRIVIAL(debug) << __func__ << " " << formattedCmd;
auto res = query(formattedCmd.c_str());
if (!res || res.status() != PGRES_COPY_IN)
{
std::stringstream ss;
@@ -284,7 +286,8 @@ Pg::bulkInsert(char const* table, std::string const& records)
{
std::stringstream ss;
ss << "bulkInsert to " << table
<< ". PQputCopyEnd status not PGRES_COMMAND_OK: " << status;
<< ". PQputCopyEnd status not PGRES_COMMAND_OK: " << status
<< " message = " << PQerrorMessage(conn_.get());
disconnect();
BOOST_LOG_TRIVIAL(error) << __func__ << " " << records;
throw std::runtime_error(ss.str());
@@ -347,11 +350,27 @@ PgPool::PgPool(boost::json::object const& config)
*/
constexpr std::size_t maxFieldSize = 1024;
constexpr std::size_t maxFields = 1000;
std::string conninfo = "postgres://";
auto getFieldAsString = [&config](auto field) {
if (!config.contains(field))
throw std::runtime_error(
field + std::string{" missing from postgres config"});
if (!config.at(field).is_string())
throw std::runtime_error(
field + std::string{" in postgres config is not a string"});
return std::string{config.at(field).as_string().c_str()};
};
conninfo += getFieldAsString("username");
conninfo += ":";
conninfo += getFieldAsString("password");
conninfo += "@";
conninfo += getFieldAsString("contact_point");
conninfo += "/";
conninfo += getFieldAsString("database");
// The connection object must be freed using the libpq API PQfinish() call.
pg_connection_type conn(
PQconnectdb(config.at("conninfo").as_string().c_str()),
[](PGconn* conn) { PQfinish(conn); });
PQconnectdb(conninfo.c_str()), [](PGconn* conn) { PQfinish(conn); });
if (!conn)
throw std::runtime_error("Can't create DB connection.");
if (PQstatus(conn.get()) != CONNECTION_OK)
@@ -602,9 +621,26 @@ PgPool::checkin(std::unique_ptr<Pg>& pg)
std::shared_ptr<PgPool>
make_PgPool(boost::json::object const& config)
{
auto ret = std::make_shared<PgPool>(config);
ret->setup();
return ret;
try
{
auto ret = std::make_shared<PgPool>(config);
ret->setup();
return ret;
}
catch (std::runtime_error& e)
{
boost::json::object configCopy = config;
configCopy["database"] = "postgres";
auto ret = std::make_shared<PgPool>(configCopy);
ret->setup();
PgQuery pgQuery{ret};
std::string query = "CREATE DATABASE " +
std::string{config.at("database").as_string().c_str()};
pgQuery(query.c_str());
ret = std::make_shared<PgPool>(config);
ret->setup();
return ret;
}
}
//-----------------------------------------------------------------------------
@@ -750,11 +786,12 @@ CREATE TABLE IF NOT EXISTS ledgers (
CREATE TABLE IF NOT EXISTS objects (
key bytea NOT NULL,
ledger_seq bigint NOT NULL,
object bytea,
PRIMARY KEY(key, ledger_seq)
ledger_seq bigint NOT NULL REFERENCES ledgers ON DELETE CASCADE,
object bytea
) PARTITION BY RANGE (ledger_seq);
CREATE INDEX objects_idx ON objects USING btree(key,ledger_seq);
create table if not exists objects1 partition of objects for values from (0) to (10000000);
create table if not exists objects2 partition of objects for values from (10000000) to (20000000);
create table if not exists objects3 partition of objects for values from (20000000) to (30000000);
@@ -772,7 +809,7 @@ CREATE INDEX IF NOT EXISTS ledgers_ledger_hash_idx ON ledgers
-- cascade here based on ledger_seq.
CREATE TABLE IF NOT EXISTS transactions (
hash bytea NOT NULL,
ledger_seq bigint NOT NULL ,
ledger_seq bigint NOT NULL REFERENCES ledgers ON DELETE CASCADE,
transaction bytea NOT NULL,
metadata bytea NOT NULL
) PARTITION BY RANGE(ledger_seq);
@@ -791,7 +828,7 @@ create index if not exists tx_by_lgr_seq on transactions using hash (ledger_seq)
-- ledger table cascade here based on ledger_seq.
CREATE TABLE IF NOT EXISTS account_transactions (
account bytea NOT NULL,
ledger_seq bigint NOT NULL ,
ledger_seq bigint NOT NULL REFERENCES ledgers ON DELETE CASCADE,
transaction_index bigint NOT NULL,
hash bytea NOT NULL,
PRIMARY KEY (account, ledger_seq, transaction_index, hash)
@@ -804,23 +841,13 @@ create table if not exists account_transactions5 partition of account_transactio
create table if not exists account_transactions6 partition of account_transactions for values from (50000000) to (60000000);
create table if not exists account_transactions7 partition of account_transactions for values from (60000000) to (70000000);
-- Table that maps a book to a list of offers in that book. Deletes from the ledger table
-- cascade here based on ledger_seq.
CREATE TABLE IF NOT EXISTS books (
ledger_seq bigint NOT NULL,
book bytea NOT NULL,
offer_key bytea NOT NULL
);
CREATE INDEX book_idx ON books using btree(ledger_seq, book, offer_key);
CREATE TABLE IF NOT EXISTS keys (
ledger_seq bigint NOT NULL,
key bytea NOT NULL
ledger_seq bigint NOT NULL,
key bytea NOT NULL,
PRIMARY KEY(ledger_seq, key)
);
CREATE INDEX key_idx ON keys USING btree(ledger_seq, key);
-- account_tx() RPC helper. From the rippled reporting process, only the
-- parameters without defaults are required. For the parameters with
-- defaults, validation should be done by rippled, such as:
@@ -937,8 +964,6 @@ CREATE OR REPLACE RULE account_transactions_update_protect AS ON UPDATE TO
account_transactions DO INSTEAD NOTHING;
CREATE OR REPLACE RULE objects_update_protect AS ON UPDATE TO
objects DO INSTEAD NOTHING;
CREATE OR REPLACE RULE books_update_protect AS ON UPDATE TO
books DO INSTEAD NOTHING;
-- Return the earliest ledger sequence intended for range operations

4
reporting/Pg.h
View File

@@ -476,6 +476,10 @@ public:
pool_->checkin(pg_);
}
// TODO. add sendQuery and getResult, for sending the query and getting the
// result asynchronously. This could be useful for sending a bunch of
// requests concurrently
/** Execute postgres query with parameters.
*
* @param dbParams Database command with parameters.

420
reporting/PostgresBackend.cpp
View File

@@ -324,7 +324,7 @@ PostgresBackend::fetchAllTransactionHashesInLedger(
}
LedgerPage
PostgresBackend::fetchLedgerPage(
PostgresBackend::doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const
@@ -335,9 +335,10 @@ PostgresBackend::fetchLedgerPage(
PgQuery pgQuery(pgPool_);
pgQuery("SET statement_timeout TO 10000");
std::stringstream sql;
sql << "SELECT key FROM keys WHERE ledger_seq = " << std::to_string(*index);
sql << "SELECT key FROM keys WHERE ledger_seq = "
<< std::to_string(index->keyIndex);
if (cursor)
sql << " AND key > \'\\x" << ripple::strHex(*cursor) << "\'";
sql << " AND key >= \'\\x" << ripple::strHex(*cursor) << "\'";
sql << " ORDER BY key ASC LIMIT " << std::to_string(limit);
BOOST_LOG_TRIVIAL(debug) << __func__ << sql.str();
auto res = pgQuery(sql.str().data());
@@ -350,8 +351,11 @@ PostgresBackend::fetchLedgerPage(
{
keys.push_back({res.asUInt256(i, 0)});
}
if (numRows == limit)
if (numRows >= limit)
{
returnCursor = keys.back();
++(*returnCursor);
}
auto objs = fetchLedgerObjects(keys, ledgerSequence);
std::vector<LedgerObject> results;
@@ -371,168 +375,6 @@ PostgresBackend::fetchLedgerPage(
return {};
}
BookOffersPage
PostgresBackend::fetchBookOffers(
ripple::uint256 const& book,
uint32_t ledgerSequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor) const
{
auto rng = fetchLedgerRange();
auto limitTuningFactor = 50;
if(!rng)
return {{},{}};
ripple::uint256 bookBase =
ripple::keylet::quality({ripple::ltDIR_NODE, book}, 0).key;
ripple::uint256 bookEnd = ripple::getQualityNext(bookBase);
using bookKeyPair = std::pair<ripple::uint256, ripple::uint256>;
auto getBooks =
[this, &bookBase, &bookEnd, &limit, &limitTuningFactor]
(std::uint32_t sequence)
-> std::pair<bool, std::vector<bookKeyPair>>
{
BOOST_LOG_TRIVIAL(info) << __func__ << ": Fetching books between "
<< "0x" << ripple::strHex(bookBase) << " and "
<< "0x" << ripple::strHex(bookEnd) << "at ledger "
<< std::to_string(sequence);
auto start = std::chrono::system_clock::now();
std::stringstream sql;
sql << "SELECT COUNT(*) FROM books WHERE "
<< "book = \'\\x" << ripple::strHex(ripple::uint256(beast::zero))
<< "\' AND ledger_seq = " << std::to_string(sequence);
bool complete;
PgQuery pgQuery(this->pgPool_);
auto res = pgQuery(sql.str().data());
if (size_t numRows = checkResult(res, 1))
complete = res.asInt(0, 0) != 0;
else
return {false, {}};
sql.str("");
sql << "SELECT book, offer_key FROM books "
<< "WHERE ledger_seq = " << std::to_string(sequence)
<< " AND book >= "
<< "\'\\x" << ripple::strHex(bookBase) << "\' "
<< "AND book < "
<< "\'\\x" << ripple::strHex(bookEnd) << "\' "
<< "ORDER BY book ASC "
<< "LIMIT " << std::to_string(limit * limitTuningFactor);
BOOST_LOG_TRIVIAL(debug) << sql.str();
res = pgQuery(sql.str().data());
auto end = std::chrono::system_clock::now();
auto duration = ((end - start).count()) / 1000000000.0;
BOOST_LOG_TRIVIAL(info) << "Postgres book key fetch took "
<< std::to_string(duration)
<< " seconds";
if (size_t numRows = checkResult(res, 2))
{
std::vector<bookKeyPair> results(numRows);
for (size_t i = 0; i < numRows; ++i)
{
auto book = res.asUInt256(i, 0);
auto key = res.asUInt256(i, 1);
results.push_back({std::move(book), std::move(key)});
}
return {complete, results};
}
return {complete, {}};
};
auto fetchObjects =
[this]
(std::vector<bookKeyPair> const& pairs,
std::uint32_t sequence,
std::uint32_t limit,
std::optional<std::string> warning)
-> BookOffersPage
{
std::vector<ripple::uint256> allKeys(pairs.size());
for (auto const& pair : pairs)
allKeys.push_back(pair.second);
auto uniqEnd = std::unique(allKeys.begin(), allKeys.end());
std::vector<ripple::uint256> keys{allKeys.begin(), uniqEnd};
auto start = std::chrono::system_clock::now();
auto ledgerEntries = fetchLedgerObjects(keys, sequence);
auto end = std::chrono::system_clock::now();
auto duration = ((end - start).count()) / 1000000000.0;
BOOST_LOG_TRIVIAL(info) << "Postgres book objects fetch took "
<< std::to_string(duration)
<< " seconds. "
<< "Fetched "
<< std::to_string(ledgerEntries.size())
<< " ledger entries";
std::vector<LedgerObject> objects;
for (auto i = 0; i < ledgerEntries.size(); ++i)
{
if(ledgerEntries[i].size() != 0)
objects.push_back(LedgerObject{keys[i], ledgerEntries[i]});
}
return {objects, {}, warning};
};
std::uint32_t bookShift = indexer_.getBookShift();
auto upper = indexer_.getBookIndexOfSeq(ledgerSequence);
auto [upperComplete, upperResults] = getBooks(upper);
BOOST_LOG_TRIVIAL(info) << __func__ << ": Upper results found "
<< upperResults.size() << " books.";
if (upperComplete)
{
BOOST_LOG_TRIVIAL(info) << "Upper book page is complete";
return fetchObjects(upperResults, ledgerSequence, limit, {});
}
BOOST_LOG_TRIVIAL(info) << "Upper book page is not complete "
<< "fetching again";
auto lower = upper - (1 << bookShift);
if (lower < rng->minSequence)
lower = rng->minSequence;
auto [lowerComplete, lowerResults] = getBooks(lower);
BOOST_LOG_TRIVIAL(info) << __func__ << ": Lower results found "
<< lowerResults.size() << " books.";
assert(lowerComplete);
std::vector<bookKeyPair> pairs;
pairs.reserve(upperResults.size() + lowerResults.size());
std::merge(upperResults.begin(), upperResults.end(),
lowerResults.begin(), lowerResults.end(),
std::back_inserter(pairs),
[](bookKeyPair pair1, bookKeyPair pair2) -> bool
{
return pair1.first < pair2.first;
});
std::optional<std::string> warning = "book data may be incomplete";
return fetchObjects(pairs, ledgerSequence, limit, warning);
}
std::vector<TransactionAndMetadata>
PostgresBackend::fetchTransactions(
std::vector<ripple::uint256> const& hashes) const
@@ -778,12 +620,16 @@ PostgresBackend::doFinishWrites() const
{
if (!abortWrite_)
{
writeConnection_.bulkInsert("transactions", transactionsBuffer_.str());
std::string txStr = transactionsBuffer_.str();
writeConnection_.bulkInsert("transactions", txStr);
writeConnection_.bulkInsert(
"account_transactions", accountTxBuffer_.str());
std::string objectsStr = objectsBuffer_.str();
if (objectsStr.size())
writeConnection_.bulkInsert("objects", objectsStr);
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " objects size = " << objectsStr.size()
<< " txns size = " << txStr.size();
}
auto res = writeConnection_("COMMIT");
if (!res || res.status() != PGRES_COMMAND_OK)
@@ -796,8 +642,6 @@ PostgresBackend::doFinishWrites() const
transactionsBuffer_.clear();
objectsBuffer_.str("");
objectsBuffer_.clear();
booksBuffer_.str("");
booksBuffer_.clear();
accountTxBuffer_.str("");
accountTxBuffer_.clear();
numRowsInObjectsBuffer_ = 0;
@@ -806,173 +650,141 @@ PostgresBackend::doFinishWrites() const
bool
PostgresBackend::writeKeys(
std::unordered_set<ripple::uint256> const& keys,
uint32_t ledgerSequence,
KeyIndex const& index,
bool isAsync) const
{
BOOST_LOG_TRIVIAL(debug) << __func__;
if (abortWrite_)
return false;
PgQuery pgQuery(pgPool_);
pgQuery("BEGIN");
std::stringstream keysBuffer;
PgQuery& conn = isAsync ? pgQuery : writeConnection_;
std::stringstream sql;
size_t numRows = 0;
for (auto& key : keys)
{
keysBuffer << std::to_string(ledgerSequence) << '\t' << "\\\\x"
<< ripple::strHex(key) << '\n';
numRows++;
// If the buffer gets too large, the insert fails. Not sure why. So we
// insert after 1 million records
if (numRows == 100000)
sql << "INSERT INTO keys (ledger_seq, key) VALUES ("
<< std::to_string(index.keyIndex) << ", \'\\x"
<< ripple::strHex(key) << "\') ON CONFLICT DO NOTHING; ";
if (numRows > 10000)
{
pgQuery.bulkInsert("keys", keysBuffer.str());
std::stringstream temp;
keysBuffer.swap(temp);
conn(sql.str().c_str());
sql.str("");
sql.clear();
numRows = 0;
}
}
if (numRows > 0)
{
pgQuery.bulkInsert("keys", keysBuffer.str());
}
pgQuery("COMMIT");
conn(sql.str().c_str());
return true;
}
bool
PostgresBackend::writeBooks(
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books,
uint32_t ledgerSequence,
bool isAsync) const
{
/*
BOOST_LOG_TRIVIAL(debug) << __func__;
std::condition_variable cv;
std::mutex mtx;
std::atomic_uint numRemaining = keys.size();
auto start = std::chrono::system_clock::now();
for (auto& key : keys)
{
boost::asio::post(
pool_, [this, key, &numRemaining, &cv, &mtx, &index]() {
PgQuery pgQuery(pgPool_);
std::stringstream sql;
sql << "INSERT INTO keys (ledger_seq, key) VALUES ("
<< std::to_string(index.keyIndex) << ", \'\\x"
<< ripple::strHex(key) << "\') ON CONFLICT DO NOTHING";
PgQuery pgQuery(pgPool_);
pgQuery("BEGIN");
std::stringstream booksBuffer;
size_t numRows = 0;
for (auto& book : books)
{
for (auto& offer : book.second)
{
booksBuffer << std::to_string(ledgerSequence) << '\t' << "\\\\x"
<< ripple::strHex(book.first) << '\t' << "\\\\x"
<< ripple::strHex(offer) << '\n';
numRows++;
// If the buffer gets too large, the insert fails. Not sure why. So
// we insert after 1 million records
if (numRows == 1000000)
{
pgQuery.bulkInsert("books", booksBuffer.str());
std::stringstream temp;
booksBuffer.swap(temp);
numRows = 0;
}
}
auto res = pgQuery(sql.str().data());
if (--numRemaining == 0)
{
std::unique_lock lck(mtx);
cv.notify_one();
}
});
}
if (numRows > 0)
{
pgQuery.bulkInsert("books", booksBuffer.str());
}
pgQuery("COMMIT");
std::unique_lock lck(mtx);
cv.wait(lck, [&numRemaining]() { return numRemaining == 0; });
auto end = std::chrono::system_clock::now();
auto duration =
std::chrono::duration_cast<std::chrono::milliseconds>(end - start)
.count();
BOOST_LOG_TRIVIAL(info)
<< __func__ << " wrote " << std::to_string(keys.size())
<< " keys with threadpool. took " << std::to_string(duration);
*/
return true;
}
bool
PostgresBackend::doOnlineDelete(uint32_t minLedgerToKeep) const
PostgresBackend::doOnlineDelete(uint32_t numLedgersToKeep) const
{
auto rng = fetchLedgerRangeNoThrow();
if (!rng)
return false;
uint32_t minLedger = rng->maxSequence - numLedgersToKeep;
if (minLedger <= rng->minSequence)
return false;
uint32_t limit = 2048;
PgQuery pgQuery(pgPool_);
pgQuery("SET statement_timeout TO 0");
std::optional<ripple::uint256> cursor;
while (true)
{
try
{
auto [objects, curCursor, warning] =
fetchLedgerPage(cursor, minLedger, 256);
if (warning)
{
BOOST_LOG_TRIVIAL(warning) << __func__
<< " online delete running but "
"flag ledger is not complete";
std::this_thread::sleep_for(std::chrono::seconds(10));
continue;
}
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
std::stringstream objectsBuffer;
for (auto& obj : objects)
{
objectsBuffer << "\\\\x" << ripple::strHex(obj.key) << '\t'
<< std::to_string(minLedger) << '\t' << "\\\\x"
<< ripple::strHex(obj.blob) << '\n';
}
pgQuery.bulkInsert("objects", objectsBuffer.str());
cursor = curCursor;
if (!cursor)
break;
}
catch (DatabaseTimeout const& e)
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " Database timeout fetching keys";
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
BOOST_LOG_TRIVIAL(info) << __func__ << " finished inserting into objects";
{
std::stringstream sql;
sql << "DELETE FROM ledgers WHERE ledger_seq < "
<< std::to_string(minLedgerToKeep);
<< std::to_string(minLedger);
auto res = pgQuery(sql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from ledgers table");
}
std::string cursor;
do
{
std::stringstream sql;
sql << "SELECT DISTINCT ON (key) key,ledger_seq,object FROM objects"
<< " WHERE ledger_seq <= " << std::to_string(minLedgerToKeep);
if (cursor.size())
sql << " AND key < \'\\x" << cursor << "\'";
sql << " ORDER BY key DESC, ledger_seq DESC"
<< " LIMIT " << std::to_string(limit);
BOOST_LOG_TRIVIAL(trace) << __func__ << sql.str();
sql << "DELETE FROM keys WHERE ledger_seq < "
<< std::to_string(minLedger);
auto res = pgQuery(sql.str().data());
BOOST_LOG_TRIVIAL(debug) << __func__ << "Fetched a page";
if (size_t numRows = checkResult(res, 3))
{
std::stringstream deleteSql;
std::stringstream deleteOffersSql;
deleteSql << "DELETE FROM objects WHERE (";
deleteOffersSql << "DELETE FROM books WHERE (";
bool firstOffer = true;
for (size_t i = 0; i < numRows; ++i)
{
std::string_view keyView{res.c_str(i, 0) + 2};
int64_t sequence = res.asBigInt(i, 1);
std::string_view objView{res.c_str(i, 2) + 2};
if (i != 0)
deleteSql << " OR ";
deleteSql << "(key = "
<< "\'\\x" << keyView << "\'";
if (objView.size() == 0)
deleteSql << " AND ledger_seq <= "
<< std::to_string(sequence);
else
deleteSql << " AND ledger_seq < "
<< std::to_string(sequence);
deleteSql << ")";
bool deleteOffer = false;
if (objView.size())
{
deleteOffer = isOfferHex(objView);
}
else
{
// This is rather unelegant. For a deleted object, we
// don't know its type just from the key (or do we?).
// So, we just assume it is an offer and try to delete
// it. The alternative is to read the actual object out
// of the db from before it was deleted. This could
// result in a lot of individual reads though, so we
// chose to just delete
deleteOffer = true;
}
if (deleteOffer)
{
if (!firstOffer)
deleteOffersSql << " OR ";
deleteOffersSql << "( offer_key = "
<< "\'\\x" << keyView << "\')";
firstOffer = false;
}
}
if (numRows == limit)
cursor = res.c_str(numRows - 1, 0) + 2;
else
cursor = {};
deleteSql << ")";
deleteOffersSql << ")";
BOOST_LOG_TRIVIAL(trace) << __func__ << deleteSql.str();
res = pgQuery(deleteSql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from objects table");
if (!firstOffer)
{
BOOST_LOG_TRIVIAL(trace) << __func__ << deleteOffersSql.str();
res = pgQuery(deleteOffersSql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from books table");
}
BOOST_LOG_TRIVIAL(debug)
<< __func__ << "Deleted a page. Cursor = " << cursor;
}
} while (cursor.size());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from keys table");
}
{
std::stringstream sql;
sql << "DELETE FROM books WHERE ledger_seq < "
<< std::to_string(minLedger);
auto res = pgQuery(sql.str().data());
if (res.msg() != "ok")
throw std::runtime_error("Error deleting from books table");
}
return true;
}

24
reporting/PostgresBackend.h
View File

@@ -9,13 +9,13 @@ class PostgresBackend : public BackendInterface
private:
mutable size_t numRowsInObjectsBuffer_ = 0;
mutable std::stringstream objectsBuffer_;
mutable std::stringstream keysBuffer_;
mutable std::stringstream transactionsBuffer_;
mutable std::stringstream booksBuffer_;
mutable std::stringstream accountTxBuffer_;
std::shared_ptr<PgPool> pgPool_;
mutable PgQuery writeConnection_;
mutable bool abortWrite_ = false;
mutable boost::asio::thread_pool pool_{200};
mutable boost::asio::thread_pool pool_{16};
uint32_t writeInterval_ = 1000000;
public:
@@ -45,18 +45,11 @@ public:
fetchAllTransactionHashesInLedger(uint32_t ledgerSequence) const override;
LedgerPage
fetchLedgerPage(
doFetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
std::uint32_t limit) const override;
BookOffersPage
fetchBookOffers(
ripple::uint256 const& book,
uint32_t ledgerSequence,
std::uint32_t limit,
std::optional<ripple::uint256> const& cursor) const override;
std::vector<TransactionAndMetadata>
fetchTransactions(
std::vector<ripple::uint256> const& hashes) const override;
@@ -113,18 +106,11 @@ public:
doFinishWrites() const override;
bool
doOnlineDelete(uint32_t minLedgerToKeep) const override;
doOnlineDelete(uint32_t numLedgersToKeep) const override;
bool
writeKeys(
std::unordered_set<ripple::uint256> const& keys,
uint32_t ledgerSequence,
bool isAsync = false) const override;
bool
writeBooks(
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books,
uint32_t ledgerSequence,
KeyIndex const& index,
bool isAsync = false) const override;
};
} // namespace Backend

86
reporting/ReportingETL.cpp
View File

@@ -239,6 +239,13 @@ ReportingETL::publishLedger(uint32_t ledgerSequence, uint32_t maxAttempts)
++numAttempts;
continue;
}
else
{
auto lgr =
flatMapBackend_->fetchLedgerBySequence(ledgerSequence);
assert(lgr);
publishLedger(*lgr);
}
}
catch (Backend::DatabaseTimeout const& e)
{
@@ -291,7 +298,7 @@ ReportingETL::fetchLedgerDataAndDiff(uint32_t idx)
std::pair<ripple::LedgerInfo, bool>
ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
{
BOOST_LOG_TRIVIAL(trace) << __func__ << " : "
BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
<< "Beginning ledger update";
ripple::LedgerInfo lgrInfo =
@@ -302,15 +309,16 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
<< "Deserialized ledger header. " << detail::toString(lgrInfo);
backend_->startWrites();
<<<<<<< HEAD
backend_->writeLedger(
=======
BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
<< "started writes";
flatMapBackend_->writeLedger(
>>>>>>> dev
lgrInfo, std::move(*rawData.mutable_ledger_header()));
std::vector<AccountTransactionsData> accountTxData{
insertTransactions(lgrInfo, rawData)};
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " : "
<< "Inserted all transactions. Number of transactions = "
<< rawData.transactions_list().transactions_size();
BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
<< "wrote ledger header";
for (auto& obj : *(rawData.mutable_ledger_objects()->mutable_objects()))
{
@@ -343,7 +351,24 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
isDeleted,
std::move(bookDir));
}
<<<<<<< HEAD
backend_->writeAccountTransactions(std::move(accountTxData));
=======
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " : "
<< "wrote objects. num objects = "
<< std::to_string(rawData.ledger_objects().objects_size());
std::vector<AccountTransactionsData> accountTxData{
insertTransactions(lgrInfo, rawData)};
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " : "
<< "Inserted all transactions. Number of transactions = "
<< rawData.transactions_list().transactions_size();
flatMapBackend_->writeAccountTransactions(std::move(accountTxData));
BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
<< "wrote account_tx";
>>>>>>> dev
accumTxns_ += rawData.transactions_list().transactions_size();
bool success = true;
if (accumTxns_ >= txnThreshold_)
@@ -353,7 +378,7 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
auto end = std::chrono::system_clock::now();
auto duration = ((end - start).count()) / 1000000000.0;
BOOST_LOG_TRIVIAL(info)
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " Accumulated " << std::to_string(accumTxns_)
<< " transactions. Wrote in " << std::to_string(duration)
<< " transactions per second = "
@@ -361,7 +386,7 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
accumTxns_ = 0;
}
else
BOOST_LOG_TRIVIAL(info) << __func__ << " skipping commit";
BOOST_LOG_TRIVIAL(debug) << __func__ << " skipping commit";
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " : "
<< "Inserted/modified/deleted all objects. Number of objects = "
@@ -412,10 +437,14 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
assert(false);
throw std::runtime_error("runETLPipeline: parent ledger is null");
}
std::atomic<uint32_t> minSequence = rng->minSequence;
BOOST_LOG_TRIVIAL(info) << __func__ << " : "
<< "Populating caches";
<<<<<<< HEAD
backend_->getIndexer().populateCachesAsync(*backend_);
=======
>>>>>>> dev
BOOST_LOG_TRIVIAL(info) << __func__ << " : "
<< "Populated caches";
@@ -501,6 +530,7 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
}
std::thread transformer{[this,
&minSequence,
&writeConflict,
&startSequence,
&getNext,
@@ -549,17 +579,25 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
lastPublishedSequence = lgrInfo.seq;
}
writeConflict = !success;
<<<<<<< HEAD
auto range = backend_->fetchLedgerRangeNoThrow();
=======
>>>>>>> dev
if (onlineDeleteInterval_ && !deleting_ &&
range->maxSequence - range->minSequence >
*onlineDeleteInterval_)
lgrInfo.seq - minSequence > *onlineDeleteInterval_)
{
deleting_ = true;
ioContext_.post([this, &range]() {
ioContext_.post([this, &minSequence]() {
BOOST_LOG_TRIVIAL(info) << "Running online delete";
<<<<<<< HEAD
backend_->doOnlineDelete(
range->maxSequence - *onlineDeleteInterval_);
=======
flatMapBackend_->doOnlineDelete(*onlineDeleteInterval_);
>>>>>>> dev
BOOST_LOG_TRIVIAL(info) << "Finished online delete";
auto rng = flatMapBackend_->fetchLedgerRangeNoThrow();
minSequence = rng->minSequence;
deleting_ = false;
});
}
@@ -580,7 +618,10 @@ ReportingETL::runETLPipeline(uint32_t startSequence, int numExtractors)
<< "Extracted and wrote " << *lastPublishedSequence - startSequence
<< " in " << ((end - begin).count()) / 1000000000.0;
writing_ = false;
<<<<<<< HEAD
backend_->getIndexer().clearCaches();
=======
>>>>>>> dev
BOOST_LOG_TRIVIAL(debug) << __func__ << " : "
<< "Stopping etl pipeline";
@@ -772,10 +813,27 @@ ReportingETL::ReportingETL(
if (config.contains("read_only"))
readOnly_ = config.at("read_only").as_bool();
if (config.contains("online_delete"))
onlineDeleteInterval_ = config.at("online_delete").as_int64();
{
int64_t interval = config.at("online_delete").as_int64();
uint32_t max = std::numeric_limits<uint32_t>::max();
if (interval > max)
{
std::stringstream msg;
msg << "online_delete cannot be greater than "
<< std::to_string(max);
throw std::runtime_error(msg.str());
}
if (interval > 0)
onlineDeleteInterval_ = static_cast<uint32_t>(interval);
}
if (config.contains("extractor_threads"))
extractorThreads_ = config.at("extractor_threads").as_int64();
if (config.contains("txn_threshold"))
txnThreshold_ = config.at("txn_threshold").as_int64();
<<<<<<< HEAD
=======
flatMapBackend_->open(readOnly_);
flatMapBackend_->checkFlagLedgers();
>>>>>>> dev
}

22
reporting/ReportingETL.h
View File

@@ -271,6 +271,28 @@ private:
return numMarkers_;
}
<<<<<<< HEAD
=======
boost::json::object
getInfo()
{
boost::json::object result;
result["etl_sources"] = loadBalancer_.toJson();
result["is_writer"] = writing_.load();
result["read_only"] = readOnly_;
auto last = getLastPublish();
if (last.time_since_epoch().count() != 0)
result["last_publish_time"] = std::to_string(
std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::system_clock::now() - getLastPublish())
.count());
return result;
}
/// start all of the necessary components and begin ETL
>>>>>>> dev
void
run()
{

22
reporting/server/listener.h
View File

@@ -39,6 +39,7 @@ class listener : public std::enable_shared_from_this<listener>
std::shared_ptr<BackendInterface> backend_;
std::shared_ptr<SubscriptionManager> subscriptions_;
std::shared_ptr<ETLLoadBalancer> balancer_;
DOSGuard& dosGuard_;
public:
static void
@@ -50,12 +51,8 @@ public:
std::shared_ptr<ETLLoadBalancer> balancer)
{
std::make_shared<listener>(
ioc,
endpoint,
backend,
subscriptions,
balancer
)->run();
ioc, endpoint, backend, subscriptions, balancer)
->run();
}
listener(
@@ -63,7 +60,8 @@ public:
boost::asio::ip::tcp::endpoint endpoint,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<ETLLoadBalancer> balancer)
std::shared_ptr<ETLLoadBalancer> balancer,
DOSGuard& dosGuard)
: ioc_(ioc)
, acceptor_(ioc)
, backend_(backend)
@@ -108,7 +106,6 @@ public:
~listener() = default;
private:
void
run()
{
@@ -134,7 +131,12 @@ private:
}
else
{
session::make_session(std::move(socket), backend_, subscriptions_, balancer_);
session::make_session(
std::move(socket),
backend_,
subscriptions_,
balancer_,
dosGuard_);
}
// Accept another connection
@@ -142,4 +144,4 @@ private:
}
};
#endif // LISTENER_H
#endif // LISTENER_H

107
reporting/server/session.cpp
View File

@@ -1,5 +1,5 @@
#include <reporting/server/session.h>
#include <reporting/P2pProxy.h>
#include <reporting/server/session.h>
void
fail(boost::beast::error_code ec, char const* what)
@@ -7,7 +7,7 @@ fail(boost::beast::error_code ec, char const* what)
std::cerr << what << ": " << ec.message() << "\n";
}
boost::json::object
std::pair<boost::json::object, uint32_t>
buildResponse(
boost::json::object const& request,
std::shared_ptr<BackendInterface> backend,
@@ -25,41 +25,86 @@ buildResponse(
switch (commandMap[command])
{
case tx:
return doTx(request, *backend);
case account_tx:
return doAccountTx(request, *backend);
case ledger:
return doLedger(request, *backend);
return {doTx(request, *backend), 1};
case account_tx: {
auto res = doAccountTx(request, backend);
if (res.contains("transactions"))
return {res, res["transactions"].as_array().size()};
return {res, 1};
}
case ledger: {
auto res = doLedger(request, backend);
if (res.contains("transactions"))
return {res, res["transactions"].as_array().size()};
return {res, 1};
}
case ledger_entry:
return doLedgerEntry(request, *backend);
return {doLedgerEntry(request, *backend), 1};
case ledger_range:
return doLedgerRange(request, *backend);
case ledger_data:
return doLedgerData(request, *backend);
return {doLedgerRange(request, *backend), 1};
case ledger_data: {
auto res = doLedgerData(request, backend);
if (res.contains("objects"))
return {res, res["objects"].as_array().size() * 4};
return {res, 1};
}
case account_info:
return doAccountInfo(request, *backend);
case book_offers:
return doBookOffers(request, *backend);
case account_channels:
return doAccountChannels(request, *backend);
case account_lines:
return doAccountLines(request, *backend);
case account_currencies:
return doAccountCurrencies(request, *backend);
case account_offers:
return doAccountOffers(request, *backend);
case account_objects:
return doAccountObjects(request, *backend);
case channel_authorize:
return doChannelAuthorize(request);
return {doAccountInfo(request, *backend), 1};
case book_offers: {
auto res = doBookOffers(request, backend);
if (res.contains("offers"))
return {res, res["offers"].as_array().size() * 4};
return {res, 1};
}
case account_channels: {
auto res = doAccountChannels(request, *backend);
if (res.contains("channels"))
return {res, res["channels"].as_array().size()};
return {res, 1};
}
case account_lines: {
auto res = doAccountLines(request, *backend);
if (res.contains("lines"))
return {res, res["lines"].as_array().size()};
return {res, 1};
}
case account_currencies: {
auto res = doAccountCurrencies(request, *backend);
size_t count = 1;
if (res.contains("send_currencies"))
count = res["send_currencies"].as_array().size();
if(res.contains("receive_currencies"]))
count += res["receive_currencies"].as_array().size();
return {res, count};
}
case account_offers: {
auto res = doAccountOffers(request, *backend);
if (res.contains("offers"))
return {res, res["offers"].as_array().size()};
return {res, 1};
}
case account_objects: {
auto res = doAccountObjects(request, *backend);
if (res.contains("objects"))
return {res, res["objects"].as_array().size()};
return {res, 1};
}
case channel_authorize: {
return {doChannelAuthorize(request), 1};
};
case channel_verify:
return doChannelVerify(request);
return {doChannelVerify(request), 1};
case subscribe:
return doSubscribe(request, session, *manager);
return {doSubscribe(request, session, *manager), 1};
case unsubscribe:
return doUnsubscribe(request, session, *manager);
return {doUnsubscribe(request, session, *manager), 1};
case server_info: {
return {doServerInfo(request, backend), 1};
break;
}
default:
response["error"] = "Unknown command: " + command;
return response;
return {response, 1};
}
}
}

76
reporting/server/session.h
View File

@@ -49,6 +49,7 @@ enum RPCCommand {
account_objects,
channel_authorize,
channel_verify,
server_info,
subscribe,
unsubscribe
};
@@ -69,6 +70,7 @@ static std::unordered_map<std::string, RPCCommand> commandMap{
{"account_objects", account_objects},
{"channel_authorize", channel_authorize},
{"channel_verify", channel_verify},
{"server_info", server_info},
{"subscribe", subscribe},
{"unsubscribe", unsubscribe}};
@@ -170,6 +172,7 @@ class session : public std::enable_shared_from_this<session>
std::shared_ptr<BackendInterface> backend_;
std::weak_ptr<SubscriptionManager> subscriptions_;
std::shared_ptr<ETLLoadBalancer> balancer_;
DOSGuard& dosGuard_;
public:
// Take ownership of the socket
@@ -177,11 +180,13 @@ public:
boost::asio::ip::tcp::socket&& socket,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<ETLLoadBalancer> balancer)
std::shared_ptr<ETLLoadBalancer> balancer,
DOSGuard& dosGuard)
: ws_(std::move(socket))
, backend_(backend)
, subscriptions_(subscriptions)
, balancer_(balancer)
, dosGuard_(dosGuard)
{
}
@@ -190,10 +195,11 @@ public:
boost::asio::ip::tcp::socket&& socket,
std::shared_ptr<BackendInterface> backend,
std::shared_ptr<SubscriptionManager> subscriptions,
std::shared_ptr<ETLLoadBalancer> balancer)
std::shared_ptr<ETLLoadBalancer> balancer,
DOSGuard& dosGuard)
{
std::make_shared<session>(
std::move(socket), backend, subscriptions, balancer)
std::move(socket), backend, subscriptions, balancer, dosGuard)
->run();
}
@@ -295,33 +301,59 @@ private:
static_cast<char const*>(buffer_.data().data()), buffer_.size()};
// BOOST_LOG_TRIVIAL(debug) << __func__ << msg;
boost::json::object response;
try
auto ip =
ws_.next_layer().socket().remote_endpoint().address().to_string();
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " received request from ip = " << ip;
if (!dosGuard_.isOk(ip))
response["error"] = "Too many requests. Slow down";
else
{
boost::json::value raw = boost::json::parse(msg);
boost::json::object request = raw.as_object();
BOOST_LOG_TRIVIAL(debug) << " received request : " << request;
try
{
std::shared_ptr<SubscriptionManager> subPtr =
subscriptions_.lock();
if (!subPtr)
return;
boost::json::value raw = boost::json::parse(msg);
boost::json::object request = raw.as_object();
BOOST_LOG_TRIVIAL(debug) << " received request : " << request;
try
{
std::shared_ptr<SubscriptionManager> subPtr =
subscriptions_.lock();
if (!subPtr)
return;
response = buildResponse(
request, backend_, subPtr, balancer_, shared_from_this());
auto [res, cost] = buildResponse(
request,
backend_,
subPtr,
balancer_,
shared_from_this());
auto start = std::chrono::system_clock::now();
response = std::move(res);
if (!dosGuard_.add(ip, cost))
{
response["warning"] = "Too many requests";
}
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__ << " RPC call took "
<< ((end - start).count() / 1000000000.0)
<< " . request = " << request;
}
catch (Backend::DatabaseTimeout const& t)
{
BOOST_LOG_TRIVIAL(error) << __func__ << " Database timeout";
response["error"] =
"Database read timeout. Please retry the request";
}
}
catch (Backend::DatabaseTimeout const& t)
catch (std::exception const& e)
{
BOOST_LOG_TRIVIAL(error) << __func__ << " Database timeout";
response["error"] =
"Database read timeout. Please retry the request";
BOOST_LOG_TRIVIAL(error)
<< __func__ << "caught exception : " << e.what();
response["error"] = "Unknown exception";
}
}
catch (std::exception const& e)
{
BOOST_LOG_TRIVIAL(error)
<< __func__ << "caught exception : " << e.what();
}
BOOST_LOG_TRIVIAL(trace) << __func__ << response;
response_ = boost::json::serialize(response);

86
server/DOSGuard.h Normal file
View File

@@ -0,0 +1,86 @@
#include <boost/asio.hpp>
#include <string>
#include <unordered_map>
#include <unordered_set>
class DOSGuard
{
std::unordered_map<std::string, uint32_t> ipFetchCount_;
uint32_t maxFetches_ = 100;
uint32_t sweepInterval_ = 1;
std::unordered_set<std::string> whitelist_;
boost::asio::io_context& ctx_;
std::mutex mtx_;
public:
DOSGuard(boost::json::object const& config, boost::asio::io_context& ctx)
: ctx_(ctx)
{
if (config.contains("dos_guard"))
{
auto dosGuardConfig = config.at("dos_guard").as_object();
if (dosGuardConfig.contains("max_fetches") &&
dosGuardConfig.contains("sweep_interval"))
{
maxFetches_ = dosGuardConfig.at("max_fetches").as_int64();
sweepInterval_ = dosGuardConfig.at("sweep_interval").as_int64();
}
if (dosGuardConfig.contains("whitelist"))
{
auto whitelist = dosGuardConfig.at("whitelist").as_array();
for (auto& ip : whitelist)
whitelist_.insert(ip.as_string().c_str());
}
}
createTimer();
}
void
createTimer()
{
auto wait = std::chrono::seconds(sweepInterval_);
std::shared_ptr<boost::asio::steady_timer> timer =
std::make_shared<boost::asio::steady_timer>(
ctx_, std::chrono::steady_clock::now() + wait);
timer->async_wait(
[timer, this](const boost::system::error_code& error) {
clear();
createTimer();
});
}
bool
isOk(std::string const& ip)
{
if (whitelist_.count(ip) > 0)
return true;
std::unique_lock lck(mtx_);
auto it = ipFetchCount_.find(ip);
if (it == ipFetchCount_.end())
return true;
return it->second < maxFetches_;
}
bool
add(std::string const& ip, uint32_t numObjects)
{
if (whitelist_.count(ip) > 0)
return true;
{
std::unique_lock lck(mtx_);
auto it = ipFetchCount_.find(ip);
if (it == ipFetchCount_.end())
ipFetchCount_[ip] = numObjects;
else
it->second += numObjects;
}
return isOk(ip);
}
void
clear()
{
std::unique_lock lck(mtx_);
ipFetchCount_.clear();
}
};

563
server/websocket_server_async.cpp Normal file
View File

@@ -0,0 +1,563 @@
//
// Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Official repository: https://github.com/boostorg/beast
//
//------------------------------------------------------------------------------
//
// Example: WebSocket server, asynchronous
//
//------------------------------------------------------------------------------
#include <boost/asio/dispatch.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include <boost/beast/websocket.hpp>
#include <boost/json.hpp>
#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/log/trivial.hpp>
#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <functional>
#include <iostream>
#include <memory>
#include <reporting/BackendFactory.h>
#include <reporting/ReportingETL.h>
#include <reporting/server/listener.h>
#include <reporting/server/session.h>
#include <server/DOSGuard.h>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
<<<<<<< HEAD:websocket_server_async.cpp
=======
//------------------------------------------------------------------------------
enum RPCCommand {
tx,
account_tx,
ledger,
account_info,
ledger_data,
book_offers,
ledger_range,
ledger_entry,
server_info
};
std::unordered_map<std::string, RPCCommand> commandMap{
{"tx", tx},
{"account_tx", account_tx},
{"ledger", ledger},
{"ledger_range", ledger_range},
{"ledger_entry", ledger_entry},
{"account_info", account_info},
{"ledger_data", ledger_data},
{"book_offers", book_offers},
{"server_info", server_info}};
boost::json::object
doAccountInfo(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doTx(boost::json::object const& request, BackendInterface const& backend);
boost::json::object
doAccountTx(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doLedgerData(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doLedgerEntry(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doBookOffers(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doLedger(boost::json::object const& request, BackendInterface const& backend);
boost::json::object
doLedgerRange(
boost::json::object const& request,
BackendInterface const& backend);
boost::json::object
doServerInfo(
boost::json::object const& request,
BackendInterface const& backend);
std::pair<boost::json::object, uint32_t>
buildResponse(
boost::json::object const& request,
BackendInterface const& backend)
{
std::string command = request.at("command").as_string().c_str();
BOOST_LOG_TRIVIAL(info) << "Received rpc command : " << request;
boost::json::object response;
switch (commandMap[command])
{
case tx:
return {doTx(request, backend), 1};
break;
case account_tx: {
auto res = doAccountTx(request, backend);
if (res.contains("transactions"))
return {res, res["transactions"].as_array().size()};
return {res, 1};
}
break;
case ledger: {
auto res = doLedger(request, backend);
if (res.contains("transactions"))
return {res, res["transactions"].as_array().size()};
return {res, 1};
}
break;
case ledger_entry:
return {doLedgerEntry(request, backend), 1};
break;
case ledger_range:
return {doLedgerRange(request, backend), 1};
break;
case ledger_data: {
auto res = doLedgerData(request, backend);
if (res.contains("objects"))
return {res, res["objects"].as_array().size()};
return {res, 1};
}
break;
case server_info: {
return {doServerInfo(request, backend), 1};
break;
}
case account_info:
return {doAccountInfo(request, backend), 1};
break;
case book_offers: {
auto res = doBookOffers(request, backend);
if (res.contains("offers"))
return {res, res["offers"].as_array().size()};
return {res, 1};
}
break;
default:
BOOST_LOG_TRIVIAL(error) << "Unknown command: " << command;
}
return {response, 1};
}
// Report a failure
void
fail(boost::beast::error_code ec, char const* what)
{
std::cerr << what << ": " << ec.message() << "\n";
}
// Echoes back all received WebSocket messages
class session : public std::enable_shared_from_this<session>
{
boost::beast::websocket::stream<boost::beast::tcp_stream> ws_;
boost::beast::flat_buffer buffer_;
std::string response_;
BackendInterface const& backend_;
DOSGuard& dosGuard_;
public:
// Take ownership of the socket
explicit session(
boost::asio::ip::tcp::socket&& socket,
BackendInterface const& backend,
DOSGuard& dosGuard)
: ws_(std::move(socket)), backend_(backend), dosGuard_(dosGuard)
{
}
// Get on the correct executor
void
run()
{
// We need to be executing within a strand to perform async
// operations on the I/O objects in this session. Although not
// strictly necessary for single-threaded contexts, this example
// code is written to be thread-safe by default.
boost::asio::dispatch(
ws_.get_executor(),
boost::beast::bind_front_handler(
&session::on_run, shared_from_this()));
}
// Start the asynchronous operation
void
on_run()
{
// Set suggested timeout settings for the websocket
ws_.set_option(boost::beast::websocket::stream_base::timeout::suggested(
boost::beast::role_type::server));
// Set a decorator to change the Server of the handshake
ws_.set_option(boost::beast::websocket::stream_base::decorator(
[](boost::beast::websocket::response_type& res) {
res.set(
boost::beast::http::field::server,
std::string(BOOST_BEAST_VERSION_STRING) +
" websocket-server-async");
}));
// Accept the websocket handshake
ws_.async_accept(boost::beast::bind_front_handler(
&session::on_accept, shared_from_this()));
}
void
on_accept(boost::beast::error_code ec)
{
if (ec)
return fail(ec, "accept");
// Read a message
do_read();
}
void
do_read()
{
// Read a message into our buffer
ws_.async_read(
buffer_,
boost::beast::bind_front_handler(
&session::on_read, shared_from_this()));
}
void
on_read(boost::beast::error_code ec, std::size_t bytes_transferred)
{
boost::ignore_unused(bytes_transferred);
// This indicates that the session was closed
if (ec == boost::beast::websocket::error::closed)
return;
if (ec)
fail(ec, "read");
std::string msg{
static_cast<char const*>(buffer_.data().data()), buffer_.size()};
// BOOST_LOG_TRIVIAL(debug) << __func__ << msg;
boost::json::object response;
auto ip =
ws_.next_layer().socket().remote_endpoint().address().to_string();
BOOST_LOG_TRIVIAL(debug)
<< __func__ << " received request from ip = " << ip;
if (!dosGuard_.isOk(ip))
response["error"] = "Too many requests. Slow down";
else
{
try
{
boost::json::value raw = boost::json::parse(msg);
boost::json::object request = raw.as_object();
BOOST_LOG_TRIVIAL(debug) << " received request : " << request;
try
{
auto start = std::chrono::system_clock::now();
auto [res, cost] = buildResponse(request, backend_);
response = std::move(res);
if (!dosGuard_.add(ip, cost))
{
response["warning"] = "Too many requests";
}
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__ << " RPC call took "
<< ((end - start).count() / 1000000000.0)
<< " . request = " << request;
}
catch (Backend::DatabaseTimeout const& t)
{
BOOST_LOG_TRIVIAL(error) << __func__ << " Database timeout";
response["error"] =
"Database read timeout. Please retry the request";
}
}
catch (std::exception const& e)
{
BOOST_LOG_TRIVIAL(error)
<< __func__ << "caught exception : " << e.what();
response["error"] = "Unknown exception";
}
}
BOOST_LOG_TRIVIAL(trace) << __func__ << response;
response_ = boost::json::serialize(response);
// Echo the message
ws_.text(ws_.got_text());
ws_.async_write(
boost::asio::buffer(response_),
boost::beast::bind_front_handler(
&session::on_write, shared_from_this()));
}
void
on_write(boost::beast::error_code ec, std::size_t bytes_transferred)
{
boost::ignore_unused(bytes_transferred);
if (ec)
return fail(ec, "write");
// Clear the buffer
buffer_.consume(buffer_.size());
// Do another read
do_read();
}
};
//------------------------------------------------------------------------------
// Accepts incoming connections and launches the sessions
class listener : public std::enable_shared_from_this<listener>
{
boost::asio::io_context& ioc_;
boost::asio::ip::tcp::acceptor acceptor_;
BackendInterface const& backend_;
DOSGuard& dosGuard_;
public:
listener(
boost::asio::io_context& ioc,
boost::asio::ip::tcp::endpoint endpoint,
BackendInterface const& backend,
DOSGuard& dosGuard)
: ioc_(ioc), acceptor_(ioc), backend_(backend), dosGuard_(dosGuard)
{
boost::beast::error_code ec;
// Open the acceptor
acceptor_.open(endpoint.protocol(), ec);
if (ec)
{
fail(ec, "open");
return;
}
// Allow address reuse
acceptor_.set_option(boost::asio::socket_base::reuse_address(true), ec);
if (ec)
{
fail(ec, "set_option");
return;
}
// Bind to the server address
acceptor_.bind(endpoint, ec);
if (ec)
{
fail(ec, "bind");
return;
}
// Start listening for connections
acceptor_.listen(boost::asio::socket_base::max_listen_connections, ec);
if (ec)
{
fail(ec, "listen");
return;
}
}
// Start accepting incoming connections
void
run()
{
do_accept();
}
private:
void
do_accept()
{
// The new connection gets its own strand
acceptor_.async_accept(
boost::asio::make_strand(ioc_),
boost::beast::bind_front_handler(
&listener::on_accept, shared_from_this()));
}
void
on_accept(boost::beast::error_code ec, boost::asio::ip::tcp::socket socket)
{
if (ec)
{
fail(ec, "accept");
}
else
{
// Create the session and run it
std::make_shared<session>(std::move(socket), backend_, dosGuard_)
->run();
}
// Accept another connection
do_accept();
}
};
>>>>>>> dev:server/websocket_server_async.cpp
std::optional<boost::json::object>
parse_config(const char* filename)
{
try
{
std::ifstream in(filename, std::ios::in | std::ios::binary);
if (in)
{
std::stringstream contents;
contents << in.rdbuf();
in.close();
std::cout << contents.str() << std::endl;
boost::json::value value = boost::json::parse(contents.str());
return value.as_object();
}
}
catch (std::exception const& e)
{
std::cout << e.what() << std::endl;
}
return {};
}
//------------------------------------------------------------------------------
//
void
initLogLevel(int level)
{
switch (level)
{
case 0:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::trace);
break;
case 1:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::debug);
break;
case 2:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::info);
break;
case 3:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::warning);
break;
case 4:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::error);
break;
case 5:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::fatal);
break;
default:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::info);
}
}
void
start(boost::asio::io_context& ioc, std::uint32_t numThreads)
{
std::vector<std::thread> v;
v.reserve(numThreads - 1);
for (auto i = numThreads - 1; i > 0; --i)
v.emplace_back([&ioc] { ioc.run(); });
ioc.run();
}
int
main(int argc, char* argv[])
{
// Check command line arguments.
if (argc != 5 and argc != 6)
{
std::cerr
<< "Usage: websocket-server-async <address> <port> <threads> "
"<config_file> <log level> \n"
<< "Example:\n"
<< " websocket-server-async 0.0.0.0 8080 1 config.json 2\n";
return EXIT_FAILURE;
}
auto const address = boost::asio::ip::make_address(argv[1]);
auto const port = static_cast<unsigned short>(std::atoi(argv[2]));
auto const threads = std::max<int>(1, std::atoi(argv[3]));
auto const config = parse_config(argv[4]);
if (argc > 5)
{
initLogLevel(std::atoi(argv[5]));
}
else
{
initLogLevel(2);
}
if (!config)
{
std::cerr << "couldnt parse config. Exiting..." << std::endl;
return EXIT_FAILURE;
}
// The io_context is required for all I/O
boost::asio::io_context ioc{threads};
<<<<<<< HEAD:websocket_server_async.cpp
std::shared_ptr<BackendInterface> backend{Backend::make_Backend(*config)};
std::shared_ptr<SubscriptionManager> subscriptions{
SubscriptionManager::make_SubscriptionManager()};
=======
ReportingETL etl{config.value(), ioc};
DOSGuard dosGuard{config.value(), ioc};
>>>>>>> dev:server/websocket_server_async.cpp
std::shared_ptr<NetworkValidatedLedgers> ledgers{
NetworkValidatedLedgers::make_ValidatedLedgers()};
std::shared_ptr<ETLLoadBalancer> balancer{
ETLLoadBalancer::make_ETLLoadBalancer(
*config,
ioc,
<<<<<<< HEAD:websocket_server_async.cpp
backend,
subscriptions,
ledgers)};
=======
boost::asio::ip::tcp::endpoint{address, port},
etl.getFlatMapBackend(),
dosGuard)
->run();
>>>>>>> dev:server/websocket_server_async.cpp
std::shared_ptr<ReportingETL> etl{ReportingETL::make_ReportingETL(
*config, ioc, backend, subscriptions, balancer, ledgers)};
listener::make_listener(
ioc,
boost::asio::ip::tcp::endpoint{address, port},
backend,
subscriptions,
balancer);
// Blocks until stopped.
// When stopped, shared_ptrs fall out of scope
// Calls destructors on all resources, and destructs in order
start(ioc, threads);
return EXIT_SUCCESS;
}

24
test.py
View File

@@ -436,9 +436,12 @@ async def ledger_data(ip, port, ledger, limit, binary, cursor):
address = 'ws://' + str(ip) + ':' + str(port)
try:
async with websockets.connect(address) as ws:
await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"binary":bool(binary),"limit":int(limit),"cursor":cursor}))
await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"binary":bool(binary),"cursor":cursor}))
if limit is not None:
await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"binary":bool(binary),"limit":int(limit),"cursor":cursor}))
else:
await ws.send(json.dumps({"command":"ledger_data","ledger_index":int(ledger),"binary":bool(binary),"cursor":cursor}))
res = json.loads(await ws.recv())
print(res)
objects = []
blobs = []
keys = []
@@ -598,7 +601,7 @@ async def book_offers(ip, port, ledger, pay_currency, pay_issuer, get_currency,
req["cursor"] = cursor
await ws.send(json.dumps(req))
res = json.loads(await ws.recv())
#print(json.dumps(res,indent=4,sort_keys=True))
print(json.dumps(res,indent=4,sort_keys=True))
if "result" in res:
res = res["result"]
for x in res["offers"]:
@@ -729,7 +732,7 @@ async def ledger(ip, port, ledger, binary, transactions, expand):
async with websockets.connect(address,max_size=1000000000) as ws:
await ws.send(json.dumps({"command":"ledger","ledger_index":int(ledger),"binary":bool(binary), "transactions":bool(transactions),"expand":bool(expand)}))
res = json.loads(await ws.recv())
#print(json.dumps(res,indent=4,sort_keys=True))
print(json.dumps(res,indent=4,sort_keys=True))
return res
except websockets.exceptions.connectionclosederror as e:
@@ -764,6 +767,15 @@ async def fee(ip, port):
print(json.dumps(res,indent=4,sort_keys=True))
except websockets.exceptions.connectionclosederror as e:
print(e)
async def server_info(ip, port):
address = 'ws://' + str(ip) + ':' + str(port)
try:
async with websockets.connect(address) as ws:
await ws.send(json.dumps({"command":"server_info"}))
res = json.loads(await ws.recv())
print(json.dumps(res,indent=4,sort_keys=True))
except websockets.exceptions.connectionclosederror as e:
print(e)
async def ledger_diff(ip, port, base, desired, includeBlobs):
address = 'ws://' + str(ip) + ':' + str(port)
@@ -785,7 +797,7 @@ async def perf(ip, port):
parser = argparse.ArgumentParser(description='test script for xrpl-reporting')
parser.add_argument('action', choices=["account_info", "tx", "txs","account_tx", "account_tx_full","ledger_data", "ledger_data_full", "book_offers","ledger","ledger_range","ledger_entry", "ledgers", "ledger_entries","account_txs","account_infos","account_txs_full","book_offerses","ledger_diff","perf","fee"])
parser.add_argument('action', choices=["account_info", "tx", "txs","account_tx", "account_tx_full","ledger_data", "ledger_data_full", "book_offers","ledger","ledger_range","ledger_entry", "ledgers", "ledger_entries","account_txs","account_infos","account_txs_full","book_offerses","ledger_diff","perf","fee","server_info"])
parser.add_argument('--ip', default='127.0.0.1')
parser.add_argument('--port', default='8080')
@@ -828,6 +840,8 @@ def run(args):
args.ledger = asyncio.get_event_loop().run_until_complete(ledger_range(args.ip, args.port))[1]
if args.action == "fee":
asyncio.get_event_loop().run_until_complete(fee(args.ip, args.port))
elif args.action == "server_info":
asyncio.get_event_loop().run_until_complete(server_info(args.ip, args.port))
elif args.action == "perf":
asyncio.get_event_loop().run_until_complete(
perf(args.ip,args.port))

727
unittests/main.cpp Normal file
View File

@@ -0,0 +1,727 @@
#include <algorithm>
#include <gtest/gtest.h>
#include <handlers/RPCHelpers.h>
#include <reporting/DBHelpers.h>
#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/log/trivial.hpp>
#include <reporting/BackendFactory.h>
#include <reporting/BackendInterface.h>
// Demonstrate some basic assertions.
TEST(BackendTest, Basic)
{
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::warning);
std::string keyspace =
"oceand_test_" +
std::to_string(
std::chrono::system_clock::now().time_since_epoch().count());
boost::json::object cassandraConfig{
{"database",
{{"type", "cassandra"},
{"cassandra",
{{"contact_points", "127.0.0.1"},
{"port", 9042},
{"keyspace", keyspace.c_str()},
{"replication_factor", 1},
{"table_prefix", ""},
{"max_requests_outstanding", 1000},
{"indexer_key_shift", 2},
{"threads", 8}}}}}};
boost::json::object postgresConfig{
{"database",
{{"type", "postgres"},
{"postgres",
{{"contact_point", "127.0.0.1"},
{"username", "postgres"},
{"database", keyspace.c_str()},
{"password", "postgres"},
{"indexer_key_shift", 2},
{"threads", 8}}}}}};
std::vector<boost::json::object> configs = {
cassandraConfig, postgresConfig};
for (auto& config : configs)
{
std::cout << keyspace << std::endl;
auto backend = Backend::makeBackend(config);
backend->open(false);
std::string rawHeader =
"03C3141A01633CD656F91B4EBB5EB89B791BD34DBC8A04BB6F407C5335BC54351E"
"DD73"
"3898497E809E04074D14D271E4832D7888754F9230800761563A292FA2315A6DB6"
"FE30"
"CC5909B285080FCD6773CC883F9FE0EE4D439340AC592AADB973ED3CF53E2232B3"
"3EF5"
"7CECAC2816E3122816E31A0A00F8377CD95DFA484CFAE282656A58CE5AA29652EF"
"FD80"
"AC59CD91416E4E13DBBE";
auto hexStringToBinaryString = [](auto const& hex) {
auto blob = ripple::strUnHex(hex);
std::string strBlob;
for (auto c : *blob)
{
strBlob += c;
}
return strBlob;
};
auto binaryStringToUint256 = [](auto const& bin) -> ripple::uint256 {
ripple::uint256 uint;
return uint.fromVoid((void const*)bin.data());
};
auto ledgerInfoToBinaryString = [](auto const& info) {
auto blob = ledgerInfoToBlob(info);
std::string strBlob;
for (auto c : blob)
{
strBlob += c;
}
return strBlob;
};
std::string rawHeaderBlob = hexStringToBinaryString(rawHeader);
ripple::LedgerInfo lgrInfo =
deserializeHeader(ripple::makeSlice(rawHeaderBlob));
backend->startWrites();
backend->writeLedger(lgrInfo, std::move(rawHeaderBlob), true);
ASSERT_TRUE(backend->finishWrites(lgrInfo.seq));
{
auto rng = backend->fetchLedgerRange();
EXPECT_TRUE(rng.has_value());
EXPECT_EQ(rng->minSequence, rng->maxSequence);
EXPECT_EQ(rng->maxSequence, lgrInfo.seq);
}
{
auto seq = backend->fetchLatestLedgerSequence();
EXPECT_TRUE(seq.has_value());
EXPECT_EQ(*seq, lgrInfo.seq);
}
{
auto retLgr = backend->fetchLedgerBySequence(lgrInfo.seq);
EXPECT_TRUE(retLgr.has_value());
EXPECT_EQ(retLgr->seq, lgrInfo.seq);
EXPECT_EQ(ledgerInfoToBlob(lgrInfo), ledgerInfoToBlob(*retLgr));
}
EXPECT_FALSE(
backend->fetchLedgerBySequence(lgrInfo.seq + 1).has_value());
auto lgrInfoOld = lgrInfo;
auto lgrInfoNext = lgrInfo;
lgrInfoNext.seq = lgrInfo.seq + 1;
lgrInfoNext.parentHash = lgrInfo.hash;
lgrInfoNext.hash++;
lgrInfoNext.accountHash = ~lgrInfo.accountHash;
{
std::string rawHeaderBlob = ledgerInfoToBinaryString(lgrInfoNext);
backend->startWrites();
backend->writeLedger(lgrInfoNext, std::move(rawHeaderBlob));
ASSERT_TRUE(backend->finishWrites(lgrInfoNext.seq));
}
{
auto rng = backend->fetchLedgerRange();
EXPECT_TRUE(rng.has_value());
EXPECT_EQ(rng->minSequence, lgrInfoOld.seq);
EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq);
}
{
auto seq = backend->fetchLatestLedgerSequence();
EXPECT_EQ(seq, lgrInfoNext.seq);
}
{
auto retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq);
EXPECT_TRUE(retLgr.has_value());
EXPECT_EQ(retLgr->seq, lgrInfoNext.seq);
EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoNext));
EXPECT_NE(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoOld));
retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq - 1);
EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoOld));
EXPECT_NE(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoNext));
retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq - 2);
EXPECT_FALSE(backend->fetchLedgerBySequence(lgrInfoNext.seq - 2)
.has_value());
auto txns = backend->fetchAllTransactionsInLedger(lgrInfoNext.seq);
EXPECT_EQ(txns.size(), 0);
auto hashes =
backend->fetchAllTransactionHashesInLedger(lgrInfoNext.seq);
EXPECT_EQ(hashes.size(), 0);
}
// the below dummy data is not expected to be consistent. The metadata
// string does represent valid metadata. Don't assume though that the
// transaction or its hash correspond to the metadata, or anything like
// that. These tests are purely binary tests to make sure the same data
// that goes in, comes back out
std::string metaHex =
"201C0000001AF8E411006F560A3E08122A05AC91DEFA87052B0554E4A29B46"
"3A27642EBB060B6052196592EEE72200000000240480FDB52503CE1A863300"
"000000000000003400000000000000005529983CBAED30F547471452921C3C"
"6B9F9685F292F6291000EED0A44413AF18C250101AC09600F4B502C8F7F830"
"F80B616DCB6F3970CB79AB70975A05ED5B66860B9564400000001FE217CB65"
"D54B640B31521B05000000000000000000000000434E5900000000000360E3"
"E0751BD9A566CD03FA6CAFC78118B82BA081142252F328CF91263417762570"
"D67220CCB33B1370E1E1E3110064561AC09600F4B502C8F7F830F80B616DCB"
"6F3970CB79AB70975A05ED33DF783681E8365A05ED33DF783681581AC09600"
"F4B502C8F7F830F80B616DCB6F3970CB79AB70975A05ED33DF783681031100"
"0000000000000000000000434E59000000000004110360E3E0751BD9A566CD"
"03FA6CAFC78118B82BA0E1E1E4110064561AC09600F4B502C8F7F830F80B61"
"6DCB6F3970CB79AB70975A05ED5B66860B95E72200000000365A05ED5B6686"
"0B95581AC09600F4B502C8F7F830F80B616DCB6F3970CB79AB70975A05ED5B"
"66860B95011100000000000000000000000000000000000000000211000000"
"00000000000000000000000000000000000311000000000000000000000000"
"434E59000000000004110360E3E0751BD9A566CD03FA6CAFC78118B82BA0E1"
"E1E311006F5647B05E66DE9F3DF2689E8F4CE6126D3136B6C5E79587F9D24B"
"D71A952B0852BAE8240480FDB950101AC09600F4B502C8F7F830F80B616DCB"
"6F3970CB79AB70975A05ED33DF78368164400000033C83A95F65D59D9A6291"
"9C2D18000000000000000000000000434E5900000000000360E3E0751BD9A5"
"66CD03FA6CAFC78118B82BA081142252F328CF91263417762570D67220CCB3"
"3B1370E1E1E511006456AEA3074F10FE15DAC592F8A0405C61FB7D4C98F588"
"C2D55C84718FAFBBD2604AE722000000003100000000000000003200000000"
"0000000058AEA3074F10FE15DAC592F8A0405C61FB7D4C98F588C2D55C8471"
"8FAFBBD2604A82142252F328CF91263417762570D67220CCB33B1370E1E1E5"
"1100612503CE1A8755CE935137F8C6C8DEF26B5CD93BE18105CA83F65E1E90"
"CEC546F562D25957DC0856E0311EB450B6177F969B94DBDDA83E99B7A0576A"
"CD9079573876F16C0C004F06E6240480FDB9624000000005FF0E2BE1E72200"
"000000240480FDBA2D00000005624000000005FF0E1F81142252F328CF9126"
"3417762570D67220CCB33B1370E1E1F1031000";
std::string txnHex =
"1200072200000000240480FDB920190480FDB5201B03CE1A8964400000033C"
"83A95F65D59D9A62919C2D18000000000000000000000000434E5900000000"
"000360E3E0751BD9A566CD03FA6CAFC78118B82BA068400000000000000C73"
"21022D40673B44C82DEE1DDB8B9BB53DCCE4F97B27404DB850F068DD91D685"
"E337EA7446304402202EA6B702B48B39F2197112382838F92D4C02948E9911"
"FE6B2DEBCF9183A426BC022005DAC06CD4517E86C2548A80996019F3AC60A0"
"9EED153BF60C992930D68F09F981142252F328CF91263417762570D67220CC"
"B33B1370";
std::string hashHex =
"0A81FB3D6324C2DCF73131505C6E4DC67981D7FC39F5E9574CEC4B1F22D28BF7";
// this account is not related to the above transaction and metadata
std::string accountHex =
"1100612200000000240480FDBC2503CE1A872D0000000555516931B2AD018EFFBE"
"17C5"
"C9DCCF872F36837C2C6136ACF80F2A24079CF81FD0624000000005FF0E07811422"
"52F3"
"28CF91263417762570D67220CCB33B1370";
std::string accountIndexHex =
"E0311EB450B6177F969B94DBDDA83E99B7A0576ACD9079573876F16C0C004F06";
std::string metaBlob = hexStringToBinaryString(metaHex);
std::string txnBlob = hexStringToBinaryString(txnHex);
std::string hashBlob = hexStringToBinaryString(hashHex);
std::string accountBlob = hexStringToBinaryString(accountHex);
std::string accountIndexBlob = hexStringToBinaryString(accountIndexHex);
std::vector<ripple::AccountID> affectedAccounts;
{
backend->startWrites();
lgrInfoNext.seq = lgrInfoNext.seq + 1;
lgrInfoNext.txHash = ~lgrInfo.txHash;
lgrInfoNext.accountHash =
lgrInfoNext.accountHash ^ lgrInfoNext.txHash;
lgrInfoNext.parentHash = lgrInfoNext.hash;
lgrInfoNext.hash++;
ripple::uint256 hash256;
EXPECT_TRUE(hash256.parseHex(hashHex));
ripple::TxMeta txMeta{hash256, lgrInfoNext.seq, metaBlob};
auto journal = ripple::debugLog();
auto accountsSet = txMeta.getAffectedAccounts(journal);
for (auto& a : accountsSet)
{
affectedAccounts.push_back(a);
}
std::vector<AccountTransactionsData> accountTxData;
accountTxData.emplace_back(txMeta, hash256, journal);
backend->writeLedger(
lgrInfoNext, std::move(ledgerInfoToBinaryString(lgrInfoNext)));
backend->writeTransaction(
std::move(std::string{hashBlob}),
lgrInfoNext.seq,
std::move(std::string{txnBlob}),
std::move(std::string{metaBlob}));
backend->writeAccountTransactions(std::move(accountTxData));
backend->writeLedgerObject(
std::move(std::string{accountIndexBlob}),
lgrInfoNext.seq,
std::move(std::string{accountBlob}),
true,
false,
{});
ASSERT_TRUE(backend->finishWrites(lgrInfoNext.seq));
}
{
auto rng = backend->fetchLedgerRange();
EXPECT_TRUE(rng);
EXPECT_EQ(rng->minSequence, lgrInfoOld.seq);
EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq);
auto retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq);
EXPECT_TRUE(retLgr);
EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoNext));
auto txns = backend->fetchAllTransactionsInLedger(lgrInfoNext.seq);
EXPECT_EQ(txns.size(), 1);
EXPECT_STREQ(
(const char*)txns[0].transaction.data(),
(const char*)txnBlob.data());
EXPECT_STREQ(
(const char*)txns[0].metadata.data(),
(const char*)metaBlob.data());
auto hashes =
backend->fetchAllTransactionHashesInLedger(lgrInfoNext.seq);
EXPECT_EQ(hashes.size(), 1);
EXPECT_EQ(ripple::strHex(hashes[0]), hashHex);
for (auto& a : affectedAccounts)
{
auto accountTxns = backend->fetchAccountTransactions(a, 100);
EXPECT_EQ(accountTxns.first.size(), 1);
EXPECT_EQ(accountTxns.first[0], txns[0]);
EXPECT_FALSE(accountTxns.second);
}
ripple::uint256 key256;
EXPECT_TRUE(key256.parseHex(accountIndexHex));
auto obj = backend->fetchLedgerObject(key256, lgrInfoNext.seq);
EXPECT_TRUE(obj);
EXPECT_STREQ(
(const char*)obj->data(), (const char*)accountBlob.data());
obj = backend->fetchLedgerObject(key256, lgrInfoNext.seq + 1);
EXPECT_TRUE(obj);
EXPECT_STREQ(
(const char*)obj->data(), (const char*)accountBlob.data());
obj = backend->fetchLedgerObject(key256, lgrInfoOld.seq - 1);
EXPECT_FALSE(obj);
}
// obtain a time-based seed:
unsigned seed =
std::chrono::system_clock::now().time_since_epoch().count();
std::string accountBlobOld = accountBlob;
{
backend->startWrites();
lgrInfoNext.seq = lgrInfoNext.seq + 1;
lgrInfoNext.parentHash = lgrInfoNext.hash;
lgrInfoNext.hash++;
lgrInfoNext.txHash = lgrInfoNext.txHash ^ lgrInfoNext.accountHash;
lgrInfoNext.accountHash =
~(lgrInfoNext.accountHash ^ lgrInfoNext.txHash);
backend->writeLedger(
lgrInfoNext, std::move(ledgerInfoToBinaryString(lgrInfoNext)));
std::shuffle(
accountBlob.begin(),
accountBlob.end(),
std::default_random_engine(seed));
backend->writeLedgerObject(
std::move(std::string{accountIndexBlob}),
lgrInfoNext.seq,
std::move(std::string{accountBlob}),
true,
false,
{});
ASSERT_TRUE(backend->finishWrites(lgrInfoNext.seq));
}
{
auto rng = backend->fetchLedgerRange();
EXPECT_TRUE(rng);
EXPECT_EQ(rng->minSequence, lgrInfoOld.seq);
EXPECT_EQ(rng->maxSequence, lgrInfoNext.seq);
auto retLgr = backend->fetchLedgerBySequence(lgrInfoNext.seq);
EXPECT_TRUE(retLgr);
EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfoNext));
auto txns = backend->fetchAllTransactionsInLedger(lgrInfoNext.seq);
EXPECT_EQ(txns.size(), 0);
ripple::uint256 key256;
EXPECT_TRUE(key256.parseHex(accountIndexHex));
auto obj = backend->fetchLedgerObject(key256, lgrInfoNext.seq);
EXPECT_TRUE(obj);
EXPECT_STREQ(
(const char*)obj->data(), (const char*)accountBlob.data());
obj = backend->fetchLedgerObject(key256, lgrInfoNext.seq + 1);
EXPECT_TRUE(obj);
EXPECT_STREQ(
(const char*)obj->data(), (const char*)accountBlob.data());
obj = backend->fetchLedgerObject(key256, lgrInfoNext.seq - 1);
EXPECT_TRUE(obj);
EXPECT_STREQ(
(const char*)obj->data(), (const char*)accountBlobOld.data());
obj = backend->fetchLedgerObject(key256, lgrInfoOld.seq - 1);
EXPECT_FALSE(obj);
}
auto generateObjects = [seed](
size_t numObjects, uint32_t ledgerSequence) {
std::vector<std::pair<std::string, std::string>> res{numObjects};
ripple::uint256 key;
key = ledgerSequence * 100000;
for (auto& blob : res)
{
++key;
std::string keyStr{(const char*)key.data(), key.size()};
blob.first = keyStr;
blob.second = std::to_string(ledgerSequence) + keyStr;
}
return res;
};
auto updateObjects = [](uint32_t ledgerSequence, auto objs) {
for (auto& [key, obj] : objs)
{
obj = std::to_string(ledgerSequence) + obj;
}
return objs;
};
auto generateTxns = [seed](size_t numTxns, uint32_t ledgerSequence) {
std::vector<std::tuple<std::string, std::string, std::string>> res{
numTxns};
ripple::uint256 base;
base = ledgerSequence * 100000;
for (auto& blob : res)
{
++base;
std::string hashStr{(const char*)base.data(), base.size()};
std::string txnStr =
"tx" + std::to_string(ledgerSequence) + hashStr;
std::string metaStr =
"meta" + std::to_string(ledgerSequence) + hashStr;
blob = std::make_tuple(hashStr, txnStr, metaStr);
}
return res;
};
auto generateAccounts = [](uint32_t ledgerSequence,
uint32_t numAccounts) {
std::vector<ripple::AccountID> accounts;
ripple::AccountID base;
base = ledgerSequence * 998765;
for (size_t i = 0; i < numAccounts; ++i)
{
++base;
accounts.push_back(base);
}
return accounts;
};
auto generateAccountTx = [&](uint32_t ledgerSequence, auto txns) {
std::vector<AccountTransactionsData> ret;
auto accounts = generateAccounts(ledgerSequence, 10);
std::srand(std::time(nullptr));
uint32_t idx = 0;
for (auto& [hash, txn, meta] : txns)
{
AccountTransactionsData data;
data.ledgerSequence = ledgerSequence;
data.transactionIndex = idx;
data.txHash = hash;
for (size_t i = 0; i < 3; ++i)
{
data.accounts.insert(
accounts[std::rand() % accounts.size()]);
}
++idx;
ret.push_back(data);
}
return ret;
};
auto generateNextLedger = [seed](auto lgrInfo) {
++lgrInfo.seq;
lgrInfo.parentHash = lgrInfo.hash;
std::srand(std::time(nullptr));
std::shuffle(
lgrInfo.txHash.begin(),
lgrInfo.txHash.end(),
std::default_random_engine(seed));
std::shuffle(
lgrInfo.accountHash.begin(),
lgrInfo.accountHash.end(),
std::default_random_engine(seed));
std::shuffle(
lgrInfo.hash.begin(),
lgrInfo.hash.end(),
std::default_random_engine(seed));
return lgrInfo;
};
auto writeLedger =
[&](auto lgrInfo, auto txns, auto objs, auto accountTx) {
std::cout << "writing ledger = " << std::to_string(lgrInfo.seq);
backend->startWrites();
backend->writeLedger(
lgrInfo, std::move(ledgerInfoToBinaryString(lgrInfo)));
for (auto [hash, txn, meta] : txns)
{
backend->writeTransaction(
std::move(hash),
lgrInfo.seq,
std::move(txn),
std::move(meta));
}
for (auto [key, obj] : objs)
{
std::optional<ripple::uint256> bookDir;
if (isOffer(obj.data()))
bookDir = getBook(obj);
backend->writeLedgerObject(
std::move(key),
lgrInfo.seq,
std::move(obj),
true,
false,
std::move(bookDir));
}
backend->writeAccountTransactions(std::move(accountTx));
ASSERT_TRUE(backend->finishWrites(lgrInfo.seq));
};
auto checkLedger = [&](auto lgrInfo,
auto txns,
auto objs,
auto accountTx) {
auto rng = backend->fetchLedgerRange();
auto seq = lgrInfo.seq;
EXPECT_TRUE(rng);
EXPECT_EQ(rng->minSequence, lgrInfoOld.seq);
EXPECT_GE(rng->maxSequence, seq);
auto retLgr = backend->fetchLedgerBySequence(seq);
EXPECT_TRUE(retLgr);
EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfo));
// retLgr = backend->fetchLedgerByHash(lgrInfo.hash);
// EXPECT_TRUE(retLgr);
// EXPECT_EQ(ledgerInfoToBlob(*retLgr), ledgerInfoToBlob(lgrInfo));
auto retTxns = backend->fetchAllTransactionsInLedger(seq);
for (auto [hash, txn, meta] : txns)
{
bool found = false;
for (auto [retTxn, retMeta, retSeq] : retTxns)
{
if (std::strncmp(
(const char*)retTxn.data(),
(const char*)txn.data(),
txn.size()) == 0 &&
std::strncmp(
(const char*)retMeta.data(),
(const char*)meta.data(),
meta.size()) == 0)
found = true;
}
ASSERT_TRUE(found);
}
for (auto [account, data] : accountTx)
{
std::vector<Backend::TransactionAndMetadata> retData;
std::optional<Backend::AccountTransactionsCursor> cursor;
do
{
uint32_t limit = 10;
auto res = backend->fetchAccountTransactions(
account, limit, cursor);
if (res.second)
EXPECT_EQ(res.first.size(), limit);
retData.insert(
retData.end(), res.first.begin(), res.first.end());
cursor = res.second;
} while (cursor);
EXPECT_EQ(retData.size(), data.size());
for (size_t i = 0; i < retData.size(); ++i)
{
auto [txn, meta, seq] = retData[i];
auto [hash, expTxn, expMeta] = data[i];
EXPECT_STREQ(
(const char*)txn.data(), (const char*)expTxn.data());
EXPECT_STREQ(
(const char*)meta.data(), (const char*)expMeta.data());
}
}
for (auto [key, obj] : objs)
{
auto retObj =
backend->fetchLedgerObject(binaryStringToUint256(key), seq);
if (obj.size())
{
ASSERT_TRUE(retObj.has_value());
EXPECT_STREQ(
(const char*)obj.data(), (const char*)retObj->data());
}
else
{
ASSERT_FALSE(retObj.has_value());
}
}
Backend::LedgerPage page;
std::vector<Backend::LedgerObject> retObjs;
size_t numLoops = 0;
do
{
uint32_t limit = 10;
page = backend->fetchLedgerPage(page.cursor, seq, limit);
if (page.cursor)
EXPECT_EQ(page.objects.size(), limit);
retObjs.insert(
retObjs.end(), page.objects.begin(), page.objects.end());
++numLoops;
ASSERT_FALSE(page.warning.has_value());
} while (page.cursor);
for (auto obj : objs)
{
bool found = false;
bool correct = false;
for (auto retObj : retObjs)
{
if (ripple::strHex(obj.first) == ripple::strHex(retObj.key))
{
found = true;
ASSERT_EQ(
ripple::strHex(obj.second),
ripple::strHex(retObj.blob));
}
}
ASSERT_EQ(found, obj.second.size() != 0);
}
};
std::map<uint32_t, std::vector<std::pair<std::string, std::string>>>
state;
std::map<
uint32_t,
std::vector<std::tuple<std::string, std::string, std::string>>>
allTxns;
std::unordered_map<std::string, std::pair<std::string, std::string>>
allTxnsMap;
std::
map<uint32_t, std::map<ripple::AccountID, std::vector<std::string>>>
allAccountTx;
std::map<uint32_t, ripple::LedgerInfo> lgrInfos;
for (size_t i = 0; i < 10; ++i)
{
lgrInfoNext = generateNextLedger(lgrInfoNext);
auto objs = generateObjects(25, lgrInfoNext.seq);
auto txns = generateTxns(10, lgrInfoNext.seq);
auto accountTx = generateAccountTx(lgrInfoNext.seq, txns);
for (auto rec : accountTx)
{
for (auto account : rec.accounts)
{
allAccountTx[lgrInfoNext.seq][account].push_back(
std::string{
(const char*)rec.txHash.data(), rec.txHash.size()});
}
}
EXPECT_EQ(objs.size(), 25);
EXPECT_NE(objs[0], objs[1]);
EXPECT_EQ(txns.size(), 10);
EXPECT_NE(txns[0], txns[1]);
writeLedger(lgrInfoNext, txns, objs, accountTx);
state[lgrInfoNext.seq] = objs;
allTxns[lgrInfoNext.seq] = txns;
lgrInfos[lgrInfoNext.seq] = lgrInfoNext;
for (auto& [hash, txn, meta] : txns)
{
allTxnsMap[hash] = std::make_pair(txn, meta);
}
}
std::vector<std::pair<std::string, std::string>> objs;
for (size_t i = 0; i < 10; ++i)
{
lgrInfoNext = generateNextLedger(lgrInfoNext);
if (!objs.size())
objs = generateObjects(25, lgrInfoNext.seq);
else
objs = updateObjects(lgrInfoNext.seq, objs);
auto txns = generateTxns(10, lgrInfoNext.seq);
auto accountTx = generateAccountTx(lgrInfoNext.seq, txns);
for (auto rec : accountTx)
{
for (auto account : rec.accounts)
{
allAccountTx[lgrInfoNext.seq][account].push_back(
std::string{
(const char*)rec.txHash.data(), rec.txHash.size()});
}
}
EXPECT_EQ(objs.size(), 25);
EXPECT_NE(objs[0], objs[1]);
EXPECT_EQ(txns.size(), 10);
EXPECT_NE(txns[0], txns[1]);
writeLedger(lgrInfoNext, txns, objs, accountTx);
state[lgrInfoNext.seq] = objs;
allTxns[lgrInfoNext.seq] = txns;
lgrInfos[lgrInfoNext.seq] = lgrInfoNext;
for (auto& [hash, txn, meta] : txns)
{
allTxnsMap[hash] = std::make_pair(txn, meta);
}
}
std::cout << "WROTE ALL OBJECTS" << std::endl;
auto flatten = [&](uint32_t max) {
std::vector<std::pair<std::string, std::string>> flat;
std::map<std::string, std::string> objs;
for (auto [seq, diff] : state)
{
for (auto [k, v] : diff)
{
if (seq > max)
{
if (objs.count(k) == 0)
objs[k] = "";
}
else
{
objs[k] = v;
}
}
}
for (auto [key, value] : objs)
{
flat.push_back(std::make_pair(key, value));
}
return flat;
};
auto flattenAccountTx = [&](uint32_t max) {
std::unordered_map<
ripple::AccountID,
std::vector<std::tuple<std::string, std::string, std::string>>>
accountTx;
for (auto [seq, map] : allAccountTx)
{
if (seq > max)
break;
for (auto& [account, hashes] : map)
{
for (auto& hash : hashes)
{
auto& [txn, meta] = allTxnsMap[hash];
accountTx[account].push_back(
std::make_tuple(hash, txn, meta));
}
}
}
for (auto& [account, data] : accountTx)
std::reverse(data.begin(), data.end());
return accountTx;
};
for (auto [seq, diff] : state)
{
std::cout << "flatteneing" << std::endl;
auto flat = flatten(seq);
std::cout << "flattened" << std::endl;
checkLedger(
lgrInfos[seq], allTxns[seq], flat, flattenAccountTx(seq));
std::cout << "checked" << std::endl;
}
}
}

187
websocket_server_async.cpp
View File

@@ -1,187 +0,0 @@
//
// Copyright (c) 2016-2019 Vinnie Falco (vinnie dot falco at gmail dot com)
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
//
// Official repository: https://github.com/boostorg/beast
//
//------------------------------------------------------------------------------
//
// Example: WebSocket server, asynchronous
//
//------------------------------------------------------------------------------
#include <boost/asio/dispatch.hpp>
#include <boost/asio/strand.hpp>
#include <boost/beast/core.hpp>
#include <boost/beast/websocket.hpp>
#include <boost/json.hpp>
#include <boost/log/core.hpp>
#include <boost/log/expressions.hpp>
#include <boost/log/trivial.hpp>
#include <algorithm>
#include <cstdlib>
#include <fstream>
#include <functional>
#include <iostream>
#include <memory>
#include <reporting/ReportingETL.h>
#include <reporting/BackendFactory.h>
#include <reporting/server/session.h>
#include <reporting/server/listener.h>
#include <sstream>
#include <string>
#include <thread>
#include <vector>
std::optional<boost::json::object>
parse_config(const char* filename)
{
try
{
std::ifstream in(filename, std::ios::in | std::ios::binary);
if (in)
{
std::stringstream contents;
contents << in.rdbuf();
in.close();
std::cout << contents.str() << std::endl;
boost::json::value value = boost::json::parse(contents.str());
return value.as_object();
}
}
catch (std::exception const& e)
{
std::cout << e.what() << std::endl;
}
return {};
}
//------------------------------------------------------------------------------
//
void
initLogLevel(int level)
{
switch (level)
{
case 0:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::trace);
break;
case 1:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::debug);
break;
case 2:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::info);
break;
case 3:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::warning);
break;
case 4:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::error);
break;
case 5:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::fatal);
break;
default:
boost::log::core::get()->set_filter(
boost::log::trivial::severity >= boost::log::trivial::info);
}
}
void
start(boost::asio::io_context& ioc, std::uint32_t numThreads)
{
std::vector<std::thread> v;
v.reserve(numThreads - 1);
for (auto i = numThreads - 1; i > 0; --i)
v.emplace_back([&ioc] { ioc.run(); });
ioc.run();
}
int
main(int argc, char* argv[])
{
// Check command line arguments.
if (argc != 5 and argc != 6)
{
std::cerr
<< "Usage: websocket-server-async <address> <port> <threads> "
"<config_file> <log level> \n"
<< "Example:\n"
<< " websocket-server-async 0.0.0.0 8080 1 config.json 2\n";
return EXIT_FAILURE;
}
auto const address = boost::asio::ip::make_address(argv[1]);
auto const port = static_cast<unsigned short>(std::atoi(argv[2]));
auto const threads = std::max<int>(1, std::atoi(argv[3]));
auto const config = parse_config(argv[4]);
if (argc > 5)
{
initLogLevel(std::atoi(argv[5]));
}
else
{
initLogLevel(2);
}
if (!config)
{
std::cerr << "couldnt parse config. Exiting..." << std::endl;
return EXIT_FAILURE;
}
// The io_context is required for all I/O
boost::asio::io_context ioc{threads};
std::shared_ptr<BackendInterface> backend{
Backend::make_Backend(*config)
};
std::shared_ptr<SubscriptionManager> subscriptions{
SubscriptionManager::make_SubscriptionManager()
};
std::shared_ptr<NetworkValidatedLedgers> ledgers{
NetworkValidatedLedgers::make_ValidatedLedgers()
};
std::shared_ptr<ETLLoadBalancer> balancer{ETLLoadBalancer::make_ETLLoadBalancer(
*config,
ioc,
backend,
subscriptions,
ledgers
)};
std::shared_ptr<ReportingETL> etl{ReportingETL::make_ReportingETL(
*config,
ioc,
backend,
subscriptions,
balancer,
ledgers
)};
listener::make_listener(
ioc,
boost::asio::ip::tcp::endpoint{address, port},
backend,
subscriptions,
balancer
);
// Blocks until stopped.
// When stopped, shared_ptrs fall out of scope
// Calls destructors on all resources, and destructs in order
start(ioc, threads);
return EXIT_SUCCESS;
}