ARCHIVE/clio
1
0
Fork 0
mirror of https://github.com/XRPLF/clio.git synced 2025-11-21 04:05:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix up some issues with async indexer

Browse Source
This commit is contained in:
CJ Cobb
2021-05-05 20:22:57 +00:00
parent 736e0a675f
commit 20b8059151
8 changed files with 241 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
reporting/BackendIndexer.cpp
View File

@@ -64,6 +64,66 @@ BackendIndexer::deleteBookOffer(
deletedBooks[book].insert(offerKey);
}
void
writeFlagLedger(
uint32_t ledgerSequence,
uint32_t shift,
BackendInterface const& backend,
std::unordered_set<ripple::uint256> const& keys,
std::unordered_map<
ripple::uint256,
std::unordered_set<ripple::uint256>> const& books)
{
uint32_t nextFlag = ((ledgerSequence >> shift << shift) + (1 << shift));
ripple::uint256 zero = {};
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " keys.size() = " << std::to_string(keys.size())
<< " books.size() = " << std::to_string(books.size());
while (true)
{
try
{
auto [objects, curCursor, warning] =
backend.fetchLedgerPage({}, nextFlag, 1);
if (!(warning || objects.size() == 0))
{
BOOST_LOG_TRIVIAL(warning)
<< __func__ << " flag ledger already written. sequence = "
<< std::to_string(ledgerSequence)
<< " next flag = " << std::to_string(nextFlag)
<< "returning";
return;
}
break;
}
catch (DatabaseTimeout& t)
{
;
}
}
auto start = std::chrono::system_clock::now();
backend.writeBooks(books, nextFlag);
backend.writeBooks({{zero, {zero}}}, nextFlag);
BOOST_LOG_TRIVIAL(debug) << __func__ << " wrote books. writing keys ...";
backend.writeKeys(keys, nextFlag);
backend.writeKeys({zero}, nextFlag);
auto end = std::chrono::system_clock::now();
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. ledgerSequence = " << std::to_string(ledgerSequence)
<< " nextFlag = " << std::to_string(nextFlag)
<< " keys.size() = " << std::to_string(keys.size())
<< " books.size() = " << std::to_string(books.size()) << " time = "
<< std::chrono::duration_cast<std::chrono::seconds>(end - start)
.count();
}
void
BackendIndexer::clearCaches()
{
@@ -77,9 +137,12 @@ BackendIndexer::populateCaches(
std::optional<uint32_t> sequence)
{
if (!sequence)
sequence = backend.fetchLatestLedgerSequence();
if (!sequence)
return;
{
auto rng = backend.fetchLedgerRangeNoThrow();
if (!rng)
return;
sequence = rng->maxSequence;
}
BOOST_LOG_TRIVIAL(info)
<< __func__ << " sequence = " << std::to_string(*sequence);
std::optional<ripple::uint256> cursor;
@@ -95,9 +158,9 @@ BackendIndexer::populateCaches(
<< __func__ << " performing index repair";
uint32_t lower = (*sequence - 1) >> shift_ << shift_;
populateCaches(backend, lower);
writeNext(lower, backend);
writeFlagLedger(
lower, shift_, backend, keysCumulative, booksCumulative);
clearCaches();
continue;
}
BOOST_LOG_TRIVIAL(debug) << __func__ << " fetched a page";
cursor = curCursor;
@@ -120,8 +183,8 @@ BackendIndexer::populateCaches(
std::this_thread::sleep_for(std::chrono::seconds(2));
}
}
// Do reconcilation. Remove anything from keys or books that shouldn't be
// there
// Do reconcilation. Remove anything from keys or books that shouldn't
// be there
{
std::unique_lock lck(mtx);
populatingCacheAsync = false;
@@ -163,7 +226,8 @@ BackendIndexer::populateCachesAsync(
std::unique_lock lck(mtx);
populatingCacheAsync = true;
}
BOOST_LOG_TRIVIAL(info) << __func__;
BOOST_LOG_TRIVIAL(info)
<< __func__ << " seq = " << (sequence ? std::to_string(*sequence) : "");
boost::asio::post(ioc_, [this, sequence, &backend]() {
populateCaches(backend, sequence);
});
@@ -179,56 +243,53 @@ BackendIndexer::waitForCaches()
}
void
BackendIndexer::writeNext(
BackendIndexer::writeFlagLedgerAsync(
uint32_t ledgerSequence,
BackendInterface const& backend)
{
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. sequence = " << std::to_string(ledgerSequence);
bool isFlag = (ledgerSequence % (1 << shift_)) == 0;
if (!backend.fetchLedgerRange())
{
isFlag = true;
}
if (isFlag)
{
waitForCaches();
auto booksCopy = booksCumulative;
auto keysCopy = keysCumulative;
boost::asio::post(ioc_, [=, &backend]() {
uint32_t nextSeq =
((ledgerSequence >> shift_ << shift_) + (1 << shift_));
ripple::uint256 zero = {};
BOOST_LOG_TRIVIAL(info) << __func__ << " booksCumulative.size() = "
<< std::to_string(booksCumulative.size());
backend.writeBooks(booksCopy, nextSeq);
backend.writeBooks({{zero, {zero}}}, nextSeq);
BOOST_LOG_TRIVIAL(info) << __func__ << " wrote books";
BOOST_LOG_TRIVIAL(info) << __func__ << " keysCumulative.size() = "
<< std::to_string(keysCumulative.size());
backend.writeKeys(keysCopy, nextSeq);
backend.writeKeys({zero}, nextSeq);
BOOST_LOG_TRIVIAL(info) << __func__ << " wrote keys";
});
}
waitForCaches();
auto booksCopy = booksCumulative;
auto keysCopy = keysCumulative;
boost::asio::post(ioc_, [=, this, &backend]() {
writeFlagLedger(ledgerSequence, shift_, backend, keysCopy, booksCopy);
});
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. sequence = " << std::to_string(ledgerSequence);
}
void
BackendIndexer::finish(uint32_t ledgerSequence, BackendInterface const& backend)
{
bool isFlag = ledgerSequence % (1 << shift_) == 0;
if (!backend.fetchLedgerRange())
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " starting. sequence = " << std::to_string(ledgerSequence);
bool isFirst = false;
uint32_t index = getIndexOfSeq(ledgerSequence);
auto rng = backend.fetchLedgerRangeNoThrow();
if (!rng || rng->minSequence == ledgerSequence)
{
isFlag = true;
isFirst = true;
index = ledgerSequence;
}
backend.writeKeys(keys, index);
backend.writeBooks(books, index);
if (isFirst)
{
ripple::uint256 zero = {};
backend.writeBooks({{zero, {zero}}}, ledgerSequence);
backend.writeKeys({zero}, ledgerSequence);
writeFlagLedgerAsync(ledgerSequence, backend);
}
uint32_t nextSeq = ((ledgerSequence >> shift_ << shift_) + (1 << shift_));
uint32_t curSeq = isFlag ? ledgerSequence : nextSeq;
backend.writeKeys(keys, curSeq);
keys = {};
backend.writeBooks(books, curSeq);
books = {};
BOOST_LOG_TRIVIAL(info)
<< __func__
<< " finished. sequence = " << std::to_string(ledgerSequence);
} // namespace Backend
} // namespace Backend