ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-20 11:45:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

cleanup and postgres stubs

Browse Source
This commit is contained in:
CJ Cobb
2021-02-24 14:28:15 -05:00
parent 78c6bde902
commit 009688175e
9 changed files with 42 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -57,6 +57,7 @@ include(Postgres)
target_sources(reporting PRIVATE
reporting/ETLSource.cpp
reporting/ReportingBackend.cpp
reporting/PostgresBackend.cpp
reporting/Pg.cpp
reporting/DBHelpers.cpp
reporting/ReportingETL.cpp

2
handlers/AccountTx.cpp
View File

@@ -148,7 +148,7 @@ doAccountTx(
return response;
}
std::optional<BackendInterface::AccountTransactionsCursor> cursor;
std::optional<Backend::AccountTransactionsCursor> cursor;
if (request.contains("cursor"))
{
auto const& obj = request.at("cursor").as_object();

2
handlers/BookOffers.cpp
View File

@@ -308,7 +308,7 @@ doBookOffers(
auto start = std::chrono::system_clock::now();
ripple::uint256 bookBase = getBookBase(book);
std::vector<BackendInterface::LedgerObject> offers;
std::vector<Backend::LedgerObject> offers;
if (!cursor.isZero())
{
offers = backend.fetchBookOffers(bookBase, *sequence, cursor);

4
handlers/LedgerData.cpp
View File

@@ -52,7 +52,7 @@ doLedgerData(
request.contains("binary") ? request.at("binary").as_bool() : false;
size_t limit = request.contains("limit") ? request.at("limit").as_int64()
: (binary ? 2048 : 256);
BackendInterface::LedgerPage page;
Backend::LedgerPage page;
auto start = std::chrono::system_clock::now();
page = backend.fetchLedgerPage(cursor, ledger, limit);
@@ -62,7 +62,7 @@ doLedgerData(
std::chrono::duration_cast<std::chrono::microseconds>(end - start)
.count();
boost::json::array objects;
std::vector<BackendInterface::LedgerObject>& results = page.objects;
std::vector<Backend::LedgerObject>& results = page.objects;
std::optional<ripple::uint256> const& returnedCursor = page.cursor;
BOOST_LOG_TRIVIAL(debug)
<< "doUpperBound returned " << results.size() << " results";

2
handlers/RPCHelpers.cpp
View File

@@ -22,7 +22,7 @@ accountFromStringStrict(std::string const& account)
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(BackendInterface::TransactionAndMetadata const& blobs)
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs)
{
std::pair<
std::shared_ptr<ripple::STTx const>,

2
handlers/RPCHelpers.h
View File

@@ -12,7 +12,7 @@ accountFromStringStrict(std::string const& account);
std::pair<
std::shared_ptr<ripple::STTx const>,
std::shared_ptr<ripple::STObject const>>
deserializeTxPlusMeta(BackendInterface::TransactionAndMetadata const& blobs);
deserializeTxPlusMeta(Backend::TransactionAndMetadata const& blobs);
boost::json::object
getJson(ripple::STBase const& obj);

2
reporting/ReportingBackend.cpp
View File

@@ -1,4 +1,5 @@
#include <reporting/ReportingBackend.h>
namespace Backend {
// Process the result of an asynchronous write. Retry on error
// @param fut cassandra future associated with the write
// @param cbData struct that holds the request parameters
@@ -1529,3 +1530,4 @@ CassandraFlatMapBackend::open()
}
BOOST_LOG_TRIVIAL(info) << "Opened database successfully";
}
} // namespace Backend

52
reporting/ReportingBackend.h
View File

@@ -34,6 +34,8 @@
#include <reporting/BackendInterface.h>
#include <reporting/DBHelpers.h>
namespace Backend {
void
flatMapWriteCallback(CassFuture* fut, void* cbData);
void
@@ -259,12 +261,11 @@ public:
}
std::pair<
std::vector<BackendInterface::TransactionAndMetadata>,
std::optional<BackendInterface::AccountTransactionsCursor>>
std::vector<TransactionAndMetadata>,
std::optional<AccountTransactionsCursor>>
fetchAccountTransactions(
ripple::AccountID const& account,
std::optional<BackendInterface::AccountTransactionsCursor> const&
cursor) const override
std::optional<AccountTransactionsCursor> const& cursor) const override
{
BOOST_LOG_TRIVIAL(debug) << "Starting doAccountTx";
CassStatement* statement = cass_prepared_bind(selectAccountTx_);
@@ -707,7 +708,7 @@ public:
// @param key the key of the object
// @param pno object in which to store the result
// @return result status of query
std::optional<BackendInterface::Blob>
std::optional<Blob>
fetchLedgerObject(ripple::uint256 const& key, uint32_t sequence)
const override
{
@@ -837,7 +838,7 @@ public:
return token + 1;
}
std::optional<BackendInterface::TransactionAndMetadata>
std::optional<TransactionAndMetadata>
fetchTransaction(ripple::uint256 const& hash) const override
{
BOOST_LOG_TRIVIAL(trace) << "Fetching from cassandra";
@@ -913,7 +914,7 @@ public:
<< " microseconds";
return {{txResult, metaResult}};
}
BackendInterface::LedgerPage
LedgerPage
fetchLedgerPage(
std::optional<ripple::uint256> const& cursor,
std::uint32_t ledgerSequence,
@@ -1018,8 +1019,7 @@ public:
if (keys.size())
{
std::vector<LedgerObject> results;
std::vector<BackendInterface::Blob> objs =
fetchLedgerObjects(keys, ledgerSequence);
std::vector<Blob> objs = fetchLedgerObjects(keys, ledgerSequence);
for (size_t i = 0; i < objs.size(); ++i)
{
results.push_back({keys[i], objs[i]});
@@ -1030,7 +1030,7 @@ public:
return {{}, {}};
}
std::vector<BackendInterface::LedgerObject>
std::vector<LedgerObject>
fetchBookOffers(
ripple::uint256 const& book,
uint32_t sequence,
@@ -1139,8 +1139,7 @@ public:
if (keys.size())
{
std::vector<LedgerObject> results;
std::vector<BackendInterface::Blob> objs =
fetchLedgerObjects(keys, sequence);
std::vector<Blob> objs = fetchLedgerObjects(keys, sequence);
for (size_t i = 0; i < objs.size(); ++i)
{
results.push_back({keys[i], objs[i]});
@@ -1160,7 +1159,7 @@ public:
{
CassandraFlatMapBackend const& backend;
ripple::uint256 const& hash;
BackendInterface::TransactionAndMetadata& result;
TransactionAndMetadata& result;
std::condition_variable& cv;
std::atomic_uint32_t& numFinished;
@@ -1169,7 +1168,7 @@ public:
ReadCallbackData(
CassandraFlatMapBackend const& backend,
ripple::uint256 const& hash,
BackendInterface::TransactionAndMetadata& result,
TransactionAndMetadata& result,
std::condition_variable& cv,
std::atomic_uint32_t& numFinished,
size_t batchSize)
@@ -1185,7 +1184,7 @@ public:
ReadCallbackData(ReadCallbackData const& other) = default;
};
std::vector<BackendInterface::TransactionAndMetadata>
std::vector<TransactionAndMetadata>
fetchTransactions(std::vector<ripple::uint256> const& hashes) const override
{
std::size_t const numHashes = hashes.size();
@@ -1194,8 +1193,7 @@ public:
std::atomic_uint32_t numFinished = 0;
std::condition_variable cv;
std::mutex mtx;
std::vector<BackendInterface::TransactionAndMetadata> results{
numHashes};
std::vector<TransactionAndMetadata> results{numHashes};
std::vector<std::shared_ptr<ReadCallbackData>> cbs;
cbs.reserve(numHashes);
for (std::size_t i = 0; i < hashes.size(); ++i)
@@ -1251,7 +1249,7 @@ public:
CassandraFlatMapBackend const& backend;
ripple::uint256 const& key;
uint32_t sequence;
BackendInterface::Blob& result;
Blob& result;
std::condition_variable& cv;
std::atomic_uint32_t& numFinished;
@@ -1261,7 +1259,7 @@ public:
CassandraFlatMapBackend const& backend,
ripple::uint256 const& key,
uint32_t sequence,
BackendInterface::Blob& result,
Blob& result,
std::condition_variable& cv,
std::atomic_uint32_t& numFinished,
size_t batchSize)
@@ -1277,7 +1275,7 @@ public:
ReadObjectCallbackData(ReadObjectCallbackData const& other) = default;
};
std::vector<BackendInterface::Blob>
std::vector<Blob>
fetchLedgerObjects(
std::vector<ripple::uint256> const& keys,
uint32_t sequence) const override
@@ -1288,7 +1286,7 @@ public:
std::atomic_uint32_t numFinished = 0;
std::condition_variable cv;
std::mutex mtx;
std::vector<BackendInterface::Blob> results{numKeys};
std::vector<Blob> results{numKeys};
std::vector<std::shared_ptr<ReadObjectCallbackData>> cbs;
cbs.reserve(numKeys);
for (std::size_t i = 0; i < keys.size(); ++i)
@@ -1743,13 +1741,17 @@ public:
}
void
writeAccountTransactions(AccountTransactionsData&& data) const override
writeAccountTransactions(
std::vector<AccountTransactionsData>&& data) const override
{
numRequestsOutstanding_ += data.accounts.size();
for (auto& record : data)
{
numRequestsOutstanding_ += record.accounts.size();
WriteAccountTxCallbackData* cbData =
new WriteAccountTxCallbackData(this, std::move(data));
new WriteAccountTxCallbackData(this, std::move(record));
writeAccountTx(*cbData, false);
}
}
void
writeAccountTx(WriteAccountTxCallbackData& data, bool isRetry) const
@@ -1778,6 +1780,7 @@ public:
static_cast<cass_byte_t const*>(accountData),
account.size());
if (rc != CASS_OK)
{
cass_statement_free(statement);
std::stringstream ss;
@@ -1985,4 +1988,5 @@ public:
flatMapGetCreatedCallback(CassFuture* fut, void* cbData);
};
} // namespace Backend
#endif

12
reporting/ReportingETL.cpp
View File

@@ -127,10 +127,7 @@ ReportingETL::loadInitialLedger(uint32_t startingSequence)
if (!stopping_)
{
for (auto& data : accountTxData)
{
flatMapBackend_->writeAccountTransactions(std::move(data));
}
flatMapBackend_->writeAccountTransactions(std::move(accountTxData));
bool success = flatMapBackend_->writeLedger(
lgrInfo, std::move(*ledgerData->mutable_ledger_header()));
}
@@ -301,10 +298,7 @@ ReportingETL::buildNextLedger(org::xrpl::rpc::v1::GetLedgerResponse& rawData)
isDeleted,
std::move(bookDir));
}
for (auto& data : accountTxData)
{
flatMapBackend_->writeAccountTransactions(std::move(data));
}
flatMapBackend_->writeAccountTransactions(std::move(accountTxData));
bool success = flatMapBackend_->writeLedger(
lgrInfo, std::move(*rawData.mutable_ledger_header()));
BOOST_LOG_TRIVIAL(debug)
@@ -638,7 +632,7 @@ ReportingETL::ReportingETL(
boost::asio::io_context& ioc)
: publishStrand_(ioc)
, ioContext_(ioc)
, flatMapBackend_(makeBackend(config))
, flatMapBackend_(Backend::makeBackend(config))
, pgPool_(make_PgPool(
config.at("database").as_object().at("postgres").as_object()))
, loadBalancer_(